1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // AArch64 Architecture Description File
  27 
  28 //----------REGISTER DEFINITION BLOCK------------------------------------------
  29 // This information is used by the matcher and the register allocator to
  30 // describe individual registers and classes of registers within the target
  31 // archtecture.
  32 
  33 register %{
  34 //----------Architecture Description Register Definitions----------------------
  35 // General Registers
  36 // "reg_def"  name ( register save type, C convention save type,
  37 //                   ideal register type, encoding );
  38 // Register Save Types:
  39 //
  40 // NS  = No-Save:       The register allocator assumes that these registers
  41 //                      can be used without saving upon entry to the method, &
  42 //                      that they do not need to be saved at call sites.
  43 //
  44 // SOC = Save-On-Call:  The register allocator assumes that these registers
  45 //                      can be used without saving upon entry to the method,
  46 //                      but that they must be saved at call sites.
  47 //
  48 // SOE = Save-On-Entry: The register allocator assumes that these registers
  49 //                      must be saved before using them upon entry to the
  50 //                      method, but they do not need to be saved at call
  51 //                      sites.
  52 //
  53 // AS  = Always-Save:   The register allocator assumes that these registers
  54 //                      must be saved before using them upon entry to the
  55 //                      method, & that they must be saved at call sites.
  56 //
  57 // Ideal Register Type is used to determine how to save & restore a
  58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  60 //
  61 // The encoding number is the actual bit-pattern placed into the opcodes.
  62 
  63 // We must define the 64 bit int registers in two 32 bit halves, the
  64 // real lower register and a virtual upper half register. upper halves
  65 // are used by the register allocator but are not actually supplied as
  66 // operands to memory ops.
  67 //
  68 // follow the C1 compiler in making registers
  69 //
  70 //   r0-r7,r10-r26 volatile (caller save)
  71 //   r27-r32 system (no save, no allocate)
  72 //   r8-r9 invisible to the allocator (so we can use them as scratch regs)
  73 //
  74 // as regards Java usage. we don't use any callee save registers
  75 // because this makes it difficult to de-optimise a frame (see comment
  76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
  77 //
  78 
  79 // General Registers
  80 
  81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
  82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
  83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
  84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
  85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
  86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
  87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
  88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
  89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
  90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
  91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
  92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
  93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
  94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
  95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
  96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
  97 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  98 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  99 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
 100 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 101 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
 102 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
 103 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
 104 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
 105 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
 106 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
 107 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
 108 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
 109 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
 110 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
 111 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
 112 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
 113 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()        );
 114 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
 115 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
 116 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
 117 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
 118 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
 119 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
 120 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
 121 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
 122 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
 123 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
 124 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
 125 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
 126 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
 127 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
 128 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
 129 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
 130 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
 131 reg_def R27     (  NS, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
 132 reg_def R27_H   (  NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
 133 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
 134 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
 135 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
 136 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
 137 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
 138 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
 139 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
 140 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
 141 
 142 // ----------------------------
 143 // Float/Double Registers
 144 // ----------------------------
 145 
 146 // Double Registers
 147 
 148 // The rules of ADL require that double registers be defined in pairs.
 149 // Each pair must be two 32-bit values, but not necessarily a pair of
 150 // single float registers. In each pair, ADLC-assigned register numbers
 151 // must be adjacent, with the lower number even. Finally, when the
 152 // CPU stores such a register pair to memory, the word associated with
 153 // the lower ADLC-assigned number must be stored to the lower address.
 154 
 155 // AArch64 has 32 floating-point registers. Each can store a vector of
 156 // single or double precision floating-point values up to 8 * 32
 157 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
 158 // use the first float or double element of the vector.
 159 
 160 // for Java use float registers v0-v15 are always save on call whereas
 161 // the platform ABI treats v8-v15 as callee save). float registers
 162 // v16-v31 are SOC as per the platform spec
 163 
 164   reg_def V0   ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()          );
 165   reg_def V0_H ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next()  );
 166   reg_def V0_J ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(2) );
 167   reg_def V0_K ( SOC, SOC, Op_RegF,  0, v0->as_VMReg()->next(3) );
 168 
 169   reg_def V1   ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()          );
 170   reg_def V1_H ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next()  );
 171   reg_def V1_J ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(2) );
 172   reg_def V1_K ( SOC, SOC, Op_RegF,  1, v1->as_VMReg()->next(3) );
 173 
 174   reg_def V2   ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()          );
 175   reg_def V2_H ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next()  );
 176   reg_def V2_J ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(2) );
 177   reg_def V2_K ( SOC, SOC, Op_RegF,  2, v2->as_VMReg()->next(3) );
 178 
 179   reg_def V3   ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()          );
 180   reg_def V3_H ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next()  );
 181   reg_def V3_J ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(2) );
 182   reg_def V3_K ( SOC, SOC, Op_RegF,  3, v3->as_VMReg()->next(3) );
 183 
 184   reg_def V4   ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()          );
 185   reg_def V4_H ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next()  );
 186   reg_def V4_J ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(2) );
 187   reg_def V4_K ( SOC, SOC, Op_RegF,  4, v4->as_VMReg()->next(3) );
 188 
 189   reg_def V5   ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()          );
 190   reg_def V5_H ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next()  );
 191   reg_def V5_J ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(2) );
 192   reg_def V5_K ( SOC, SOC, Op_RegF,  5, v5->as_VMReg()->next(3) );
 193 
 194   reg_def V6   ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()          );
 195   reg_def V6_H ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next()  );
 196   reg_def V6_J ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(2) );
 197   reg_def V6_K ( SOC, SOC, Op_RegF,  6, v6->as_VMReg()->next(3) );
 198 
 199   reg_def V7   ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()          );
 200   reg_def V7_H ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next()  );
 201   reg_def V7_J ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(2) );
 202   reg_def V7_K ( SOC, SOC, Op_RegF,  7, v7->as_VMReg()->next(3) );
 203 
 204   reg_def V8   ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()          );
 205   reg_def V8_H ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next()  );
 206   reg_def V8_J ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(2) );
 207   reg_def V8_K ( SOC, SOC, Op_RegF,  8, v8->as_VMReg()->next(3) );
 208 
 209   reg_def V9   ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()          );
 210   reg_def V9_H ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next()  );
 211   reg_def V9_J ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(2) );
 212   reg_def V9_K ( SOC, SOC, Op_RegF,  9, v9->as_VMReg()->next(3) );
 213 
 214   reg_def V10  ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()         );
 215   reg_def V10_H( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next() );
 216   reg_def V10_J( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2));
 217   reg_def V10_K( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3));
 218 
 219   reg_def V11  ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()         );
 220   reg_def V11_H( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next() );
 221   reg_def V11_J( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2));
 222   reg_def V11_K( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3));
 223 
 224   reg_def V12  ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()         );
 225   reg_def V12_H( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next() );
 226   reg_def V12_J( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2));
 227   reg_def V12_K( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3));
 228 
 229   reg_def V13  ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()         );
 230   reg_def V13_H( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next() );
 231   reg_def V13_J( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2));
 232   reg_def V13_K( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3));
 233 
 234   reg_def V14  ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()         );
 235   reg_def V14_H( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next() );
 236   reg_def V14_J( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2));
 237   reg_def V14_K( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3));
 238 
 239   reg_def V15  ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()         );
 240   reg_def V15_H( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next() );
 241   reg_def V15_J( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2));
 242   reg_def V15_K( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3));
 243 
 244   reg_def V16  ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()         );
 245   reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
 246   reg_def V16_J( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2));
 247   reg_def V16_K( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3));
 248 
 249   reg_def V17  ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()         );
 250   reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
 251   reg_def V17_J( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2));
 252   reg_def V17_K( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3));
 253 
 254   reg_def V18  ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()         );
 255   reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
 256   reg_def V18_J( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2));
 257   reg_def V18_K( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3));
 258 
 259   reg_def V19  ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()         );
 260   reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
 261   reg_def V19_J( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2));
 262   reg_def V19_K( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3));
 263 
 264   reg_def V20  ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()         );
 265   reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
 266   reg_def V20_J( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2));
 267   reg_def V20_K( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3));
 268 
 269   reg_def V21  ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()         );
 270   reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
 271   reg_def V21_J( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2));
 272   reg_def V21_K( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3));
 273 
 274   reg_def V22  ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()         );
 275   reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
 276   reg_def V22_J( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2));
 277   reg_def V22_K( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3));
 278 
 279   reg_def V23  ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()         );
 280   reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
 281   reg_def V23_J( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2));
 282   reg_def V23_K( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3));
 283 
 284   reg_def V24  ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()         );
 285   reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
 286   reg_def V24_J( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2));
 287   reg_def V24_K( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3));
 288 
 289   reg_def V25  ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()         );
 290   reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
 291   reg_def V25_J( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2));
 292   reg_def V25_K( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3));
 293 
 294   reg_def V26  ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()         );
 295   reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
 296   reg_def V26_J( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2));
 297   reg_def V26_K( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3));
 298 
 299   reg_def V27  ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()         );
 300   reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
 301   reg_def V27_J( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2));
 302   reg_def V27_K( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3));
 303 
 304   reg_def V28  ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()         );
 305   reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
 306   reg_def V28_J( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2));
 307   reg_def V28_K( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3));
 308 
 309   reg_def V29  ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()         );
 310   reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
 311   reg_def V29_J( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2));
 312   reg_def V29_K( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3));
 313 
 314   reg_def V30  ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()         );
 315   reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
 316   reg_def V30_J( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2));
 317   reg_def V30_K( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3));
 318 
 319   reg_def V31  ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()         );
 320   reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
 321   reg_def V31_J( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2));
 322   reg_def V31_K( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3));
 323 
 324 // ----------------------------
 325 // Special Registers
 326 // ----------------------------
 327 
 328 // the AArch64 CSPR status flag register is not directly acessible as
 329 // instruction operand. the FPSR status flag register is a system
 330 // register which can be written/read using MSR/MRS but again does not
 331 // appear as an operand (a code identifying the FSPR occurs as an
 332 // immediate value in the instruction).
 333 
 334 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
 335 
 336 
 337 // Specify priority of register selection within phases of register
 338 // allocation.  Highest priority is first.  A useful heuristic is to
 339 // give registers a low priority when they are required by machine
 340 // instructions, like EAX and EDX on I486, and choose no-save registers
 341 // before save-on-call, & save-on-call before save-on-entry.  Registers
 342 // which participate in fixed calling sequences should come last.
 343 // Registers which are used as pairs must fall on an even boundary.
 344 
 345 alloc_class chunk0(
 346     // volatiles
 347     R10, R10_H,
 348     R11, R11_H,
 349     R12, R12_H,
 350     R13, R13_H,
 351     R14, R14_H,
 352     R15, R15_H,
 353     R16, R16_H,
 354     R17, R17_H,
 355     R18, R18_H,
 356 
 357     // arg registers
 358     R0, R0_H,
 359     R1, R1_H,
 360     R2, R2_H,
 361     R3, R3_H,
 362     R4, R4_H,
 363     R5, R5_H,
 364     R6, R6_H,
 365     R7, R7_H,
 366 
 367     // non-volatiles
 368     R19, R19_H,
 369     R20, R20_H,
 370     R21, R21_H,
 371     R22, R22_H,
 372     R23, R23_H,
 373     R24, R24_H,
 374     R25, R25_H,
 375     R26, R26_H,
 376 
 377     // non-allocatable registers
 378 
 379     R27, R27_H, // heapbase
 380     R28, R28_H, // thread
 381     R29, R29_H, // fp
 382     R30, R30_H, // lr
 383     R31, R31_H, // sp
 384 );
 385 
 386 alloc_class chunk1(
 387 
 388     // no save
 389     V16, V16_H, V16_J, V16_K,
 390     V17, V17_H, V17_J, V17_K,
 391     V18, V18_H, V18_J, V18_K,
 392     V19, V19_H, V19_J, V19_K,
 393     V20, V20_H, V20_J, V20_K,
 394     V21, V21_H, V21_J, V21_K,
 395     V22, V22_H, V22_J, V22_K,
 396     V23, V23_H, V23_J, V23_K,
 397     V24, V24_H, V24_J, V24_K,
 398     V25, V25_H, V25_J, V25_K,
 399     V26, V26_H, V26_J, V26_K,
 400     V27, V27_H, V27_J, V27_K,
 401     V28, V28_H, V28_J, V28_K,
 402     V29, V29_H, V29_J, V29_K,
 403     V30, V30_H, V30_J, V30_K,
 404     V31, V31_H, V31_J, V31_K,
 405 
 406     // arg registers
 407     V0, V0_H, V0_J, V0_K,
 408     V1, V1_H, V1_J, V1_K,
 409     V2, V2_H, V2_J, V2_K,
 410     V3, V3_H, V3_J, V3_K,
 411     V4, V4_H, V4_J, V4_K,
 412     V5, V5_H, V5_J, V5_K,
 413     V6, V6_H, V6_J, V6_K,
 414     V7, V7_H, V7_J, V7_K,
 415 
 416     // non-volatiles
 417     V8, V8_H, V8_J, V8_K,
 418     V9, V9_H, V9_J, V9_K,
 419     V10, V10_H, V10_J, V10_K,
 420     V11, V11_H, V11_J, V11_K,
 421     V12, V12_H, V12_J, V12_K,
 422     V13, V13_H, V13_J, V13_K,
 423     V14, V14_H, V14_J, V14_K,
 424     V15, V15_H, V15_J, V15_K,
 425 );
 426 
 427 alloc_class chunk2(RFLAGS);
 428 
 429 //----------Architecture Description Register Classes--------------------------
 430 // Several register classes are automatically defined based upon information in
 431 // this architecture description.
 432 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 433 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 434 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 435 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 436 //
 437 
 438 // Class for all 32 bit integer registers -- excludes SP which will
 439 // never be used as an integer register
 440 reg_class any_reg32(
 441     R0,
 442     R1,
 443     R2,
 444     R3,
 445     R4,
 446     R5,
 447     R6,
 448     R7,
 449     R10,
 450     R11,
 451     R12,
 452     R13,
 453     R14,
 454     R15,
 455     R16,
 456     R17,
 457     R18,
 458     R19,
 459     R20,
 460     R21,
 461     R22,
 462     R23,
 463     R24,
 464     R25,
 465     R26,
 466     R27,
 467     R28,
 468     R29,
 469     R30
 470 );
 471 
 472 // Singleton class for R0 int register
 473 reg_class int_r0_reg(R0);
 474 
 475 // Singleton class for R2 int register
 476 reg_class int_r2_reg(R2);
 477 
 478 // Singleton class for R3 int register
 479 reg_class int_r3_reg(R3);
 480 
 481 // Singleton class for R4 int register
 482 reg_class int_r4_reg(R4);
 483 
 484 // Class for all long integer registers (including RSP)
 485 reg_class any_reg(
 486     R0, R0_H,
 487     R1, R1_H,
 488     R2, R2_H,
 489     R3, R3_H,
 490     R4, R4_H,
 491     R5, R5_H,
 492     R6, R6_H,
 493     R7, R7_H,
 494     R10, R10_H,
 495     R11, R11_H,
 496     R12, R12_H,
 497     R13, R13_H,
 498     R14, R14_H,
 499     R15, R15_H,
 500     R16, R16_H,
 501     R17, R17_H,
 502     R18, R18_H,
 503     R19, R19_H,
 504     R20, R20_H,
 505     R21, R21_H,
 506     R22, R22_H,
 507     R23, R23_H,
 508     R24, R24_H,
 509     R25, R25_H,
 510     R26, R26_H,
 511     R27, R27_H,
 512     R28, R28_H,
 513     R29, R29_H,
 514     R30, R30_H,
 515     R31, R31_H
 516 );
 517 
 518 // Class for all non-special integer registers
 519 reg_class no_special_reg32_no_fp(
 520     R0,
 521     R1,
 522     R2,
 523     R3,
 524     R4,
 525     R5,
 526     R6,
 527     R7,
 528     R10,
 529     R11,
 530     R12,                        // rmethod
 531     R13,
 532     R14,
 533     R15,
 534     R16,
 535     R17,
 536     R18,
 537     R19,
 538     R20,
 539     R21,
 540     R22,
 541     R23,
 542     R24,
 543     R25,
 544     R26
 545  /* R27, */                     // heapbase
 546  /* R28, */                     // thread
 547  /* R29, */                     // fp
 548  /* R30, */                     // lr
 549  /* R31 */                      // sp
 550 );
 551 
 552 reg_class no_special_reg32_with_fp(
 553     R0,
 554     R1,
 555     R2,
 556     R3,
 557     R4,
 558     R5,
 559     R6,
 560     R7,
 561     R10,
 562     R11,
 563     R12,                        // rmethod
 564     R13,
 565     R14,
 566     R15,
 567     R16,
 568     R17,
 569     R18,
 570     R19,
 571     R20,
 572     R21,
 573     R22,
 574     R23,
 575     R24,
 576     R25,
 577     R26
 578  /* R27, */                     // heapbase
 579  /* R28, */                     // thread
 580     R29,                        // fp
 581  /* R30, */                     // lr
 582  /* R31 */                      // sp
 583 );
 584 
 585 reg_class_dynamic no_special_reg32(no_special_reg32_no_fp, no_special_reg32_with_fp, %{ PreserveFramePointer %});
 586 
 587 // Class for all non-special long integer registers
 588 reg_class no_special_reg_no_fp(
 589     R0, R0_H,
 590     R1, R1_H,
 591     R2, R2_H,
 592     R3, R3_H,
 593     R4, R4_H,
 594     R5, R5_H,
 595     R6, R6_H,
 596     R7, R7_H,
 597     R10, R10_H,
 598     R11, R11_H,
 599     R12, R12_H,                 // rmethod
 600     R13, R13_H,
 601     R14, R14_H,
 602     R15, R15_H,
 603     R16, R16_H,
 604     R17, R17_H,
 605     R18, R18_H,
 606     R19, R19_H,
 607     R20, R20_H,
 608     R21, R21_H,
 609     R22, R22_H,
 610     R23, R23_H,
 611     R24, R24_H,
 612     R25, R25_H,
 613     R26, R26_H,
 614  /* R27, R27_H, */              // heapbase
 615  /* R28, R28_H, */              // thread
 616  /* R29, R29_H, */              // fp
 617  /* R30, R30_H, */              // lr
 618  /* R31, R31_H */               // sp
 619 );
 620 
 621 reg_class no_special_reg_with_fp(
 622     R0, R0_H,
 623     R1, R1_H,
 624     R2, R2_H,
 625     R3, R3_H,
 626     R4, R4_H,
 627     R5, R5_H,
 628     R6, R6_H,
 629     R7, R7_H,
 630     R10, R10_H,
 631     R11, R11_H,
 632     R12, R12_H,                 // rmethod
 633     R13, R13_H,
 634     R14, R14_H,
 635     R15, R15_H,
 636     R16, R16_H,
 637     R17, R17_H,
 638     R18, R18_H,
 639     R19, R19_H,
 640     R20, R20_H,
 641     R21, R21_H,
 642     R22, R22_H,
 643     R23, R23_H,
 644     R24, R24_H,
 645     R25, R25_H,
 646     R26, R26_H,
 647  /* R27, R27_H, */              // heapbase
 648  /* R28, R28_H, */              // thread
 649     R29, R29_H,                 // fp
 650  /* R30, R30_H, */              // lr
 651  /* R31, R31_H */               // sp
 652 );
 653 
 654 reg_class_dynamic no_special_reg(no_special_reg_no_fp, no_special_reg_with_fp, %{ PreserveFramePointer %});
 655 
 656 // Class for 64 bit register r0
 657 reg_class r0_reg(
 658     R0, R0_H
 659 );
 660 
 661 // Class for 64 bit register r1
 662 reg_class r1_reg(
 663     R1, R1_H
 664 );
 665 
 666 // Class for 64 bit register r2
 667 reg_class r2_reg(
 668     R2, R2_H
 669 );
 670 
 671 // Class for 64 bit register r3
 672 reg_class r3_reg(
 673     R3, R3_H
 674 );
 675 
 676 // Class for 64 bit register r4
 677 reg_class r4_reg(
 678     R4, R4_H
 679 );
 680 
 681 // Class for 64 bit register r5
 682 reg_class r5_reg(
 683     R5, R5_H
 684 );
 685 
 686 // Class for 64 bit register r10
 687 reg_class r10_reg(
 688     R10, R10_H
 689 );
 690 
 691 // Class for 64 bit register r11
 692 reg_class r11_reg(
 693     R11, R11_H
 694 );
 695 
 696 // Class for method register
 697 reg_class method_reg(
 698     R12, R12_H
 699 );
 700 
 701 // Class for heapbase register
 702 reg_class heapbase_reg(
 703     R27, R27_H
 704 );
 705 
 706 // Class for thread register
 707 reg_class thread_reg(
 708     R28, R28_H
 709 );
 710 
 711 // Class for frame pointer register
 712 reg_class fp_reg(
 713     R29, R29_H
 714 );
 715 
 716 // Class for link register
 717 reg_class lr_reg(
 718     R30, R30_H
 719 );
 720 
 721 // Class for long sp register
 722 reg_class sp_reg(
 723   R31, R31_H
 724 );
 725 
 726 // Class for all pointer registers
 727 reg_class ptr_reg(
 728     R0, R0_H,
 729     R1, R1_H,
 730     R2, R2_H,
 731     R3, R3_H,
 732     R4, R4_H,
 733     R5, R5_H,
 734     R6, R6_H,
 735     R7, R7_H,
 736     R10, R10_H,
 737     R11, R11_H,
 738     R12, R12_H,
 739     R13, R13_H,
 740     R14, R14_H,
 741     R15, R15_H,
 742     R16, R16_H,
 743     R17, R17_H,
 744     R18, R18_H,
 745     R19, R19_H,
 746     R20, R20_H,
 747     R21, R21_H,
 748     R22, R22_H,
 749     R23, R23_H,
 750     R24, R24_H,
 751     R25, R25_H,
 752     R26, R26_H,
 753     R27, R27_H,
 754     R28, R28_H,
 755     R29, R29_H,
 756     R30, R30_H,
 757     R31, R31_H
 758 );
 759 
 760 // Class for all non_special pointer registers
 761 reg_class no_special_ptr_reg(
 762     R0, R0_H,
 763     R1, R1_H,
 764     R2, R2_H,
 765     R3, R3_H,
 766     R4, R4_H,
 767     R5, R5_H,
 768     R6, R6_H,
 769     R7, R7_H,
 770     R10, R10_H,
 771     R11, R11_H,
 772     R12, R12_H,
 773     R13, R13_H,
 774     R14, R14_H,
 775     R15, R15_H,
 776     R16, R16_H,
 777     R17, R17_H,
 778     R18, R18_H,
 779     R19, R19_H,
 780     R20, R20_H,
 781     R21, R21_H,
 782     R22, R22_H,
 783     R23, R23_H,
 784     R24, R24_H,
 785     R25, R25_H,
 786     R26, R26_H,
 787  /* R27, R27_H, */              // heapbase
 788  /* R28, R28_H, */              // thread
 789  /* R29, R29_H, */              // fp
 790  /* R30, R30_H, */              // lr
 791  /* R31, R31_H */               // sp
 792 );
 793 
 794 // Class for all float registers
 795 reg_class float_reg(
 796     V0,
 797     V1,
 798     V2,
 799     V3,
 800     V4,
 801     V5,
 802     V6,
 803     V7,
 804     V8,
 805     V9,
 806     V10,
 807     V11,
 808     V12,
 809     V13,
 810     V14,
 811     V15,
 812     V16,
 813     V17,
 814     V18,
 815     V19,
 816     V20,
 817     V21,
 818     V22,
 819     V23,
 820     V24,
 821     V25,
 822     V26,
 823     V27,
 824     V28,
 825     V29,
 826     V30,
 827     V31
 828 );
 829 
 830 // Double precision float registers have virtual `high halves' that
 831 // are needed by the allocator.
 832 // Class for all double registers
 833 reg_class double_reg(
 834     V0, V0_H,
 835     V1, V1_H,
 836     V2, V2_H,
 837     V3, V3_H,
 838     V4, V4_H,
 839     V5, V5_H,
 840     V6, V6_H,
 841     V7, V7_H,
 842     V8, V8_H,
 843     V9, V9_H,
 844     V10, V10_H,
 845     V11, V11_H,
 846     V12, V12_H,
 847     V13, V13_H,
 848     V14, V14_H,
 849     V15, V15_H,
 850     V16, V16_H,
 851     V17, V17_H,
 852     V18, V18_H,
 853     V19, V19_H,
 854     V20, V20_H,
 855     V21, V21_H,
 856     V22, V22_H,
 857     V23, V23_H,
 858     V24, V24_H,
 859     V25, V25_H,
 860     V26, V26_H,
 861     V27, V27_H,
 862     V28, V28_H,
 863     V29, V29_H,
 864     V30, V30_H,
 865     V31, V31_H
 866 );
 867 
 868 // Class for all 64bit vector registers
 869 reg_class vectord_reg(
 870     V0, V0_H,
 871     V1, V1_H,
 872     V2, V2_H,
 873     V3, V3_H,
 874     V4, V4_H,
 875     V5, V5_H,
 876     V6, V6_H,
 877     V7, V7_H,
 878     V8, V8_H,
 879     V9, V9_H,
 880     V10, V10_H,
 881     V11, V11_H,
 882     V12, V12_H,
 883     V13, V13_H,
 884     V14, V14_H,
 885     V15, V15_H,
 886     V16, V16_H,
 887     V17, V17_H,
 888     V18, V18_H,
 889     V19, V19_H,
 890     V20, V20_H,
 891     V21, V21_H,
 892     V22, V22_H,
 893     V23, V23_H,
 894     V24, V24_H,
 895     V25, V25_H,
 896     V26, V26_H,
 897     V27, V27_H,
 898     V28, V28_H,
 899     V29, V29_H,
 900     V30, V30_H,
 901     V31, V31_H
 902 );
 903 
 904 // Class for all 128bit vector registers
 905 reg_class vectorx_reg(
 906     V0, V0_H, V0_J, V0_K,
 907     V1, V1_H, V1_J, V1_K,
 908     V2, V2_H, V2_J, V2_K,
 909     V3, V3_H, V3_J, V3_K,
 910     V4, V4_H, V4_J, V4_K,
 911     V5, V5_H, V5_J, V5_K,
 912     V6, V6_H, V6_J, V6_K,
 913     V7, V7_H, V7_J, V7_K,
 914     V8, V8_H, V8_J, V8_K,
 915     V9, V9_H, V9_J, V9_K,
 916     V10, V10_H, V10_J, V10_K,
 917     V11, V11_H, V11_J, V11_K,
 918     V12, V12_H, V12_J, V12_K,
 919     V13, V13_H, V13_J, V13_K,
 920     V14, V14_H, V14_J, V14_K,
 921     V15, V15_H, V15_J, V15_K,
 922     V16, V16_H, V16_J, V16_K,
 923     V17, V17_H, V17_J, V17_K,
 924     V18, V18_H, V18_J, V18_K,
 925     V19, V19_H, V19_J, V19_K,
 926     V20, V20_H, V20_J, V20_K,
 927     V21, V21_H, V21_J, V21_K,
 928     V22, V22_H, V22_J, V22_K,
 929     V23, V23_H, V23_J, V23_K,
 930     V24, V24_H, V24_J, V24_K,
 931     V25, V25_H, V25_J, V25_K,
 932     V26, V26_H, V26_J, V26_K,
 933     V27, V27_H, V27_J, V27_K,
 934     V28, V28_H, V28_J, V28_K,
 935     V29, V29_H, V29_J, V29_K,
 936     V30, V30_H, V30_J, V30_K,
 937     V31, V31_H, V31_J, V31_K
 938 );
 939 
 940 // Class for 128 bit register v0
 941 reg_class v0_reg(
 942     V0, V0_H
 943 );
 944 
 945 // Class for 128 bit register v1
 946 reg_class v1_reg(
 947     V1, V1_H
 948 );
 949 
 950 // Class for 128 bit register v2
 951 reg_class v2_reg(
 952     V2, V2_H
 953 );
 954 
 955 // Class for 128 bit register v3
 956 reg_class v3_reg(
 957     V3, V3_H
 958 );
 959 
 960 // Singleton class for condition codes
 961 reg_class int_flags(RFLAGS);
 962 
 963 %}
 964 
 965 //----------DEFINITION BLOCK---------------------------------------------------
 966 // Define name --> value mappings to inform the ADLC of an integer valued name
 967 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 968 // Format:
 969 //        int_def  <name>         ( <int_value>, <expression>);
 970 // Generated Code in ad_<arch>.hpp
 971 //        #define  <name>   (<expression>)
 972 //        // value == <int_value>
 973 // Generated code in ad_<arch>.cpp adlc_verification()
 974 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 975 //
 976 
 977 // we follow the ppc-aix port in using a simple cost model which ranks
 978 // register operations as cheap, memory ops as more expensive and
 979 // branches as most expensive. the first two have a low as well as a
 980 // normal cost. huge cost appears to be a way of saying don't do
 981 // something
 982 
 983 definitions %{
 984   // The default cost (of a register move instruction).
 985   int_def INSN_COST            (    100,     100);
 986   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 987   int_def CALL_COST            (    200,     2 * INSN_COST);
 988   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 989 %}
 990 
 991 
 992 //----------SOURCE BLOCK-------------------------------------------------------
 993 // This is a block of C++ code which provides values, functions, and
 994 // definitions necessary in the rest of the architecture description
 995 
 996 source_hpp %{
 997 
 998 #include "asm/macroAssembler.hpp"
 999 #include "gc/shared/cardTable.hpp"
1000 #include "gc/shared/cardTableBarrierSet.hpp"
1001 #include "gc/shared/collectedHeap.hpp"
1002 #include "opto/addnode.hpp"
1003 
1004 class CallStubImpl {
1005 
1006   //--------------------------------------------------------------
1007   //---<  Used for optimization in Compile::shorten_branches  >---
1008   //--------------------------------------------------------------
1009 
1010  public:
1011   // Size of call trampoline stub.
1012   static uint size_call_trampoline() {
1013     return 0; // no call trampolines on this platform
1014   }
1015 
1016   // number of relocations needed by a call trampoline stub
1017   static uint reloc_call_trampoline() {
1018     return 0; // no call trampolines on this platform
1019   }
1020 };
1021 
1022 class HandlerImpl {
1023 
1024  public:
1025 
1026   static int emit_exception_handler(CodeBuffer &cbuf);
1027   static int emit_deopt_handler(CodeBuffer& cbuf);
1028 
1029   static uint size_exception_handler() {
1030     return MacroAssembler::far_branch_size();
1031   }
1032 
1033   static uint size_deopt_handler() {
1034     // count one adr and one far branch instruction
1035     return 4 * NativeInstruction::instruction_size;
1036   }
1037 };
1038 
1039  bool is_CAS(int opcode, bool maybe_volatile);
1040 
1041   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1042 
1043   bool unnecessary_acquire(const Node *barrier);
1044   bool needs_acquiring_load(const Node *load);
1045 
1046   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1047 
1048   bool unnecessary_release(const Node *barrier);
1049   bool unnecessary_volatile(const Node *barrier);
1050   bool needs_releasing_store(const Node *store);
1051 
1052   // predicate controlling translation of CompareAndSwapX
1053   bool needs_acquiring_load_exclusive(const Node *load);
1054 
1055   // predicate controlling translation of StoreCM
1056   bool unnecessary_storestore(const Node *storecm);
1057 
1058   // predicate controlling addressing modes
1059   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1060 %}
1061 
1062 source %{
1063 
1064   // Optimizaton of volatile gets and puts
1065   // -------------------------------------
1066   //
1067   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1068   // use to implement volatile reads and writes. For a volatile read
1069   // we simply need
1070   //
1071   //   ldar<x>
1072   //
1073   // and for a volatile write we need
1074   //
1075   //   stlr<x>
1076   //
1077   // Alternatively, we can implement them by pairing a normal
1078   // load/store with a memory barrier. For a volatile read we need
1079   //
1080   //   ldr<x>
1081   //   dmb ishld
1082   //
1083   // for a volatile write
1084   //
1085   //   dmb ish
1086   //   str<x>
1087   //   dmb ish
1088   //
1089   // We can also use ldaxr and stlxr to implement compare and swap CAS
1090   // sequences. These are normally translated to an instruction
1091   // sequence like the following
1092   //
1093   //   dmb      ish
1094   // retry:
1095   //   ldxr<x>   rval raddr
1096   //   cmp       rval rold
1097   //   b.ne done
1098   //   stlxr<x>  rval, rnew, rold
1099   //   cbnz      rval retry
1100   // done:
1101   //   cset      r0, eq
1102   //   dmb ishld
1103   //
1104   // Note that the exclusive store is already using an stlxr
1105   // instruction. That is required to ensure visibility to other
1106   // threads of the exclusive write (assuming it succeeds) before that
1107   // of any subsequent writes.
1108   //
1109   // The following instruction sequence is an improvement on the above
1110   //
1111   // retry:
1112   //   ldaxr<x>  rval raddr
1113   //   cmp       rval rold
1114   //   b.ne done
1115   //   stlxr<x>  rval, rnew, rold
1116   //   cbnz      rval retry
1117   // done:
1118   //   cset      r0, eq
1119   //
1120   // We don't need the leading dmb ish since the stlxr guarantees
1121   // visibility of prior writes in the case that the swap is
1122   // successful. Crucially we don't have to worry about the case where
1123   // the swap is not successful since no valid program should be
1124   // relying on visibility of prior changes by the attempting thread
1125   // in the case where the CAS fails.
1126   //
1127   // Similarly, we don't need the trailing dmb ishld if we substitute
1128   // an ldaxr instruction since that will provide all the guarantees we
1129   // require regarding observation of changes made by other threads
1130   // before any change to the CAS address observed by the load.
1131   //
1132   // In order to generate the desired instruction sequence we need to
1133   // be able to identify specific 'signature' ideal graph node
1134   // sequences which i) occur as a translation of a volatile reads or
1135   // writes or CAS operations and ii) do not occur through any other
1136   // translation or graph transformation. We can then provide
1137   // alternative aldc matching rules which translate these node
1138   // sequences to the desired machine code sequences. Selection of the
1139   // alternative rules can be implemented by predicates which identify
1140   // the relevant node sequences.
1141   //
1142   // The ideal graph generator translates a volatile read to the node
1143   // sequence
1144   //
1145   //   LoadX[mo_acquire]
1146   //   MemBarAcquire
1147   //
1148   // As a special case when using the compressed oops optimization we
1149   // may also see this variant
1150   //
1151   //   LoadN[mo_acquire]
1152   //   DecodeN
1153   //   MemBarAcquire
1154   //
1155   // A volatile write is translated to the node sequence
1156   //
1157   //   MemBarRelease
1158   //   StoreX[mo_release] {CardMark}-optional
1159   //   MemBarVolatile
1160   //
1161   // n.b. the above node patterns are generated with a strict
1162   // 'signature' configuration of input and output dependencies (see
1163   // the predicates below for exact details). The card mark may be as
1164   // simple as a few extra nodes or, in a few GC configurations, may
1165   // include more complex control flow between the leading and
1166   // trailing memory barriers. However, whatever the card mark
1167   // configuration these signatures are unique to translated volatile
1168   // reads/stores -- they will not appear as a result of any other
1169   // bytecode translation or inlining nor as a consequence of
1170   // optimizing transforms.
1171   //
1172   // We also want to catch inlined unsafe volatile gets and puts and
1173   // be able to implement them using either ldar<x>/stlr<x> or some
1174   // combination of ldr<x>/stlr<x> and dmb instructions.
1175   //
1176   // Inlined unsafe volatiles puts manifest as a minor variant of the
1177   // normal volatile put node sequence containing an extra cpuorder
1178   // membar
1179   //
1180   //   MemBarRelease
1181   //   MemBarCPUOrder
1182   //   StoreX[mo_release] {CardMark}-optional
1183   //   MemBarCPUOrder
1184   //   MemBarVolatile
1185   //
1186   // n.b. as an aside, a cpuorder membar is not itself subject to
1187   // matching and translation by adlc rules.  However, the rule
1188   // predicates need to detect its presence in order to correctly
1189   // select the desired adlc rules.
1190   //
1191   // Inlined unsafe volatile gets manifest as a slightly different
1192   // node sequence to a normal volatile get because of the
1193   // introduction of some CPUOrder memory barriers to bracket the
1194   // Load. However, but the same basic skeleton of a LoadX feeding a
1195   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1196   // present
1197   //
1198   //   MemBarCPUOrder
1199   //        ||       \\
1200   //   MemBarCPUOrder LoadX[mo_acquire]
1201   //        ||            |
1202   //        ||       {DecodeN} optional
1203   //        ||       /
1204   //     MemBarAcquire
1205   //
1206   // In this case the acquire membar does not directly depend on the
1207   // load. However, we can be sure that the load is generated from an
1208   // inlined unsafe volatile get if we see it dependent on this unique
1209   // sequence of membar nodes. Similarly, given an acquire membar we
1210   // can know that it was added because of an inlined unsafe volatile
1211   // get if it is fed and feeds a cpuorder membar and if its feed
1212   // membar also feeds an acquiring load.
1213   //
1214   // Finally an inlined (Unsafe) CAS operation is translated to the
1215   // following ideal graph
1216   //
1217   //   MemBarRelease
1218   //   MemBarCPUOrder
1219   //   CompareAndSwapX {CardMark}-optional
1220   //   MemBarCPUOrder
1221   //   MemBarAcquire
1222   //
1223   // So, where we can identify these volatile read and write
1224   // signatures we can choose to plant either of the above two code
1225   // sequences. For a volatile read we can simply plant a normal
1226   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1227   // also choose to inhibit translation of the MemBarAcquire and
1228   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1229   //
1230   // When we recognise a volatile store signature we can choose to
1231   // plant at a dmb ish as a translation for the MemBarRelease, a
1232   // normal str<x> and then a dmb ish for the MemBarVolatile.
1233   // Alternatively, we can inhibit translation of the MemBarRelease
1234   // and MemBarVolatile and instead plant a simple stlr<x>
1235   // instruction.
1236   //
1237   // when we recognise a CAS signature we can choose to plant a dmb
1238   // ish as a translation for the MemBarRelease, the conventional
1239   // macro-instruction sequence for the CompareAndSwap node (which
1240   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1241   // Alternatively, we can elide generation of the dmb instructions
1242   // and plant the alternative CompareAndSwap macro-instruction
1243   // sequence (which uses ldaxr<x>).
1244   //
1245   // Of course, the above only applies when we see these signature
1246   // configurations. We still want to plant dmb instructions in any
1247   // other cases where we may see a MemBarAcquire, MemBarRelease or
1248   // MemBarVolatile. For example, at the end of a constructor which
1249   // writes final/volatile fields we will see a MemBarRelease
1250   // instruction and this needs a 'dmb ish' lest we risk the
1251   // constructed object being visible without making the
1252   // final/volatile field writes visible.
1253   //
1254   // n.b. the translation rules below which rely on detection of the
1255   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1256   // If we see anything other than the signature configurations we
1257   // always just translate the loads and stores to ldr<x> and str<x>
1258   // and translate acquire, release and volatile membars to the
1259   // relevant dmb instructions.
1260   //
1261 
1262   // is_CAS(int opcode, bool maybe_volatile)
1263   //
1264   // return true if opcode is one of the possible CompareAndSwapX
1265   // values otherwise false.
1266 
1267   bool is_CAS(int opcode, bool maybe_volatile)
1268   {
1269     switch(opcode) {
1270       // We handle these
1271     case Op_CompareAndSwapI:
1272     case Op_CompareAndSwapL:
1273     case Op_CompareAndSwapP:
1274     case Op_CompareAndSwapN:
1275     case Op_ShenandoahCompareAndSwapP:
1276     case Op_ShenandoahCompareAndSwapN:
1277     case Op_CompareAndSwapB:
1278     case Op_CompareAndSwapS:
1279     case Op_GetAndSetI:
1280     case Op_GetAndSetL:
1281     case Op_GetAndSetP:
1282     case Op_GetAndSetN:
1283     case Op_GetAndAddI:
1284     case Op_GetAndAddL:
1285       return true;
1286     case Op_CompareAndExchangeI:
1287     case Op_CompareAndExchangeN:
1288     case Op_CompareAndExchangeB:
1289     case Op_CompareAndExchangeS:
1290     case Op_CompareAndExchangeL:
1291     case Op_CompareAndExchangeP:
1292     case Op_WeakCompareAndSwapB:
1293     case Op_WeakCompareAndSwapS:
1294     case Op_WeakCompareAndSwapI:
1295     case Op_WeakCompareAndSwapL:
1296     case Op_WeakCompareAndSwapP:
1297     case Op_WeakCompareAndSwapN:
1298     case Op_ShenandoahWeakCompareAndSwapP:
1299     case Op_ShenandoahWeakCompareAndSwapN:
1300     case Op_ShenandoahCompareAndExchangeP:
1301     case Op_ShenandoahCompareAndExchangeN:
1302       return maybe_volatile;
1303     default:
1304       return false;
1305     }
1306   }
1307 
1308   // helper to determine the maximum number of Phi nodes we may need to
1309   // traverse when searching from a card mark membar for the merge mem
1310   // feeding a trailing membar or vice versa
1311 
1312 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1313 
1314 bool unnecessary_acquire(const Node *barrier)
1315 {
1316   assert(barrier->is_MemBar(), "expecting a membar");
1317 
1318   if (UseBarriersForVolatile) {
1319     // we need to plant a dmb
1320     return false;
1321   }
1322 
1323   MemBarNode* mb = barrier->as_MemBar();
1324 
1325   if (mb->trailing_load()) {
1326     return true;
1327   }
1328 
1329   if (mb->trailing_load_store()) {
1330     Node* load_store = mb->in(MemBarNode::Precedent);
1331     assert(load_store->is_LoadStore(), "unexpected graph shape");
1332     return is_CAS(load_store->Opcode(), true);
1333   }
1334 
1335   return false;
1336 }
1337 
1338 bool needs_acquiring_load(const Node *n)
1339 {
1340   assert(n->is_Load(), "expecting a load");
1341   if (UseBarriersForVolatile) {
1342     // we use a normal load and a dmb
1343     return false;
1344   }
1345 
1346   LoadNode *ld = n->as_Load();
1347 
1348   return ld->is_acquire();
1349 }
1350 
1351 bool unnecessary_release(const Node *n)
1352 {
1353   assert((n->is_MemBar() &&
1354           n->Opcode() == Op_MemBarRelease),
1355          "expecting a release membar");
1356 
1357   if (UseBarriersForVolatile) {
1358     // we need to plant a dmb
1359     return false;
1360   }
1361 
1362   MemBarNode *barrier = n->as_MemBar();
1363   if (!barrier->leading()) {
1364     return false;
1365   } else {
1366     Node* trailing = barrier->trailing_membar();
1367     MemBarNode* trailing_mb = trailing->as_MemBar();
1368     assert(trailing_mb->trailing(), "Not a trailing membar?");
1369     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1370 
1371     Node* mem = trailing_mb->in(MemBarNode::Precedent);
1372     if (mem->is_Store()) {
1373       assert(mem->as_Store()->is_release(), "");
1374       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1375       return true;
1376     } else {
1377       assert(mem->is_LoadStore(), "");
1378       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1379       return is_CAS(mem->Opcode(), true);
1380     }
1381   }
1382   return false;
1383 }
1384 
1385 bool unnecessary_volatile(const Node *n)
1386 {
1387   // assert n->is_MemBar();
1388   if (UseBarriersForVolatile) {
1389     // we need to plant a dmb
1390     return false;
1391   }
1392 
1393   MemBarNode *mbvol = n->as_MemBar();
1394 
1395   bool release = mbvol->trailing_store();
1396   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1397 #ifdef ASSERT
1398   if (release) {
1399     Node* leading = mbvol->leading_membar();
1400     assert(leading->Opcode() == Op_MemBarRelease, "");
1401     assert(leading->as_MemBar()->leading_store(), "");
1402     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1403   }
1404 #endif
1405 
1406   return release;
1407 }
1408 
1409 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1410 
1411 bool needs_releasing_store(const Node *n)
1412 {
1413   // assert n->is_Store();
1414   if (UseBarriersForVolatile) {
1415     // we use a normal store and dmb combination
1416     return false;
1417   }
1418 
1419   StoreNode *st = n->as_Store();
1420 
1421   return st->trailing_membar() != NULL;
1422 }
1423 
1424 // predicate controlling translation of CAS
1425 //
1426 // returns true if CAS needs to use an acquiring load otherwise false
1427 
1428 bool needs_acquiring_load_exclusive(const Node *n)
1429 {
1430   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1431   if (UseBarriersForVolatile) {
1432     return false;
1433   }
1434 
1435   LoadStoreNode* ldst = n->as_LoadStore();
1436   if (is_CAS(n->Opcode(), false)) {
1437     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1438   } else {
1439     return ldst->trailing_membar() != NULL;
1440   }
1441 
1442   // so we can just return true here
1443   return true;
1444 }
1445 
1446 // predicate controlling translation of StoreCM
1447 //
1448 // returns true if a StoreStore must precede the card write otherwise
1449 // false
1450 
1451 bool unnecessary_storestore(const Node *storecm)
1452 {
1453   assert(storecm->Opcode()  == Op_StoreCM, "expecting a StoreCM");
1454 
1455   // we need to generate a dmb ishst between an object put and the
1456   // associated card mark when we are using CMS without conditional
1457   // card marking
1458 
1459   if (UseConcMarkSweepGC && !UseCondCardMark) {
1460     return false;
1461   }
1462 
1463   // a storestore is unnecesary in all other cases
1464 
1465   return true;
1466 }
1467 
1468 
1469 #define __ _masm.
1470 
1471 // advance declarations for helper functions to convert register
1472 // indices to register objects
1473 
1474 // the ad file has to provide implementations of certain methods
1475 // expected by the generic code
1476 //
1477 // REQUIRED FUNCTIONALITY
1478 
1479 //=============================================================================
1480 
1481 // !!!!! Special hack to get all types of calls to specify the byte offset
1482 //       from the start of the call to the point where the return address
1483 //       will point.
1484 
1485 int MachCallStaticJavaNode::ret_addr_offset()
1486 {
1487   // call should be a simple bl
1488   int off = 4;
1489   return off;
1490 }
1491 
1492 int MachCallDynamicJavaNode::ret_addr_offset()
1493 {
1494   return 16; // movz, movk, movk, bl
1495 }
1496 
1497 int MachCallRuntimeNode::ret_addr_offset() {
1498   // for generated stubs the call will be
1499   //   far_call(addr)
1500   // for real runtime callouts it will be six instructions
1501   // see aarch64_enc_java_to_runtime
1502   //   adr(rscratch2, retaddr)
1503   //   lea(rscratch1, RuntimeAddress(addr)
1504   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1505   //   blrt rscratch1
1506   CodeBlob *cb = CodeCache::find_blob(_entry_point);
1507   if (cb) {
1508     return MacroAssembler::far_branch_size();
1509   } else {
1510     return 6 * NativeInstruction::instruction_size;
1511   }
1512 }
1513 
1514 // Indicate if the safepoint node needs the polling page as an input
1515 
1516 // the shared code plants the oop data at the start of the generated
1517 // code for the safepoint node and that needs ot be at the load
1518 // instruction itself. so we cannot plant a mov of the safepoint poll
1519 // address followed by a load. setting this to true means the mov is
1520 // scheduled as a prior instruction. that's better for scheduling
1521 // anyway.
1522 
1523 bool SafePointNode::needs_polling_address_input()
1524 {
1525   return true;
1526 }
1527 
1528 //=============================================================================
1529 
1530 #ifndef PRODUCT
1531 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1532   st->print("BREAKPOINT");
1533 }
1534 #endif
1535 
1536 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1537   MacroAssembler _masm(&cbuf);
1538   __ brk(0);
1539 }
1540 
1541 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1542   return MachNode::size(ra_);
1543 }
1544 
1545 //=============================================================================
1546 
1547 #ifndef PRODUCT
1548   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1549     st->print("nop \t# %d bytes pad for loops and calls", _count);
1550   }
1551 #endif
1552 
1553   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1554     MacroAssembler _masm(&cbuf);
1555     for (int i = 0; i < _count; i++) {
1556       __ nop();
1557     }
1558   }
1559 
1560   uint MachNopNode::size(PhaseRegAlloc*) const {
1561     return _count * NativeInstruction::instruction_size;
1562   }
1563 
1564 //=============================================================================
1565 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1566 
1567 int Compile::ConstantTable::calculate_table_base_offset() const {
1568   return 0;  // absolute addressing, no offset
1569 }
1570 
1571 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1572 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1573   ShouldNotReachHere();
1574 }
1575 
1576 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1577   // Empty encoding
1578 }
1579 
1580 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1581   return 0;
1582 }
1583 
1584 #ifndef PRODUCT
1585 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1586   st->print("-- \t// MachConstantBaseNode (empty encoding)");
1587 }
1588 #endif
1589 
1590 #ifndef PRODUCT
1591 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1592   Compile* C = ra_->C;
1593 
1594   int framesize = C->frame_slots() << LogBytesPerInt;
1595 
1596   if (C->need_stack_bang(framesize))
1597     st->print("# stack bang size=%d\n\t", framesize);
1598 
1599   if (framesize < ((1 << 9) + 2 * wordSize)) {
1600     st->print("sub  sp, sp, #%d\n\t", framesize);
1601     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1602     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
1603   } else {
1604     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1605     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
1606     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1607     st->print("sub  sp, sp, rscratch1");
1608   }
1609 }
1610 #endif
1611 
1612 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1613   Compile* C = ra_->C;
1614   MacroAssembler _masm(&cbuf);
1615 
1616   // n.b. frame size includes space for return pc and rfp
1617   const long framesize = C->frame_size_in_bytes();
1618   assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
1619 
1620   // insert a nop at the start of the prolog so we can patch in a
1621   // branch if we need to invalidate the method later
1622   __ nop();
1623 
1624   int bangsize = C->bang_size_in_bytes();
1625   if (C->need_stack_bang(bangsize) && UseStackBanging)
1626     __ generate_stack_overflow_check(bangsize);
1627 
1628   __ build_frame(framesize);
1629 
1630   if (NotifySimulator) {
1631     __ notify(Assembler::method_entry);
1632   }
1633 
1634   if (VerifyStackAtCalls) {
1635     Unimplemented();
1636   }
1637 
1638   C->set_frame_complete(cbuf.insts_size());
1639 
1640   if (C->has_mach_constant_base_node()) {
1641     // NOTE: We set the table base offset here because users might be
1642     // emitted before MachConstantBaseNode.
1643     Compile::ConstantTable& constant_table = C->constant_table();
1644     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1645   }
1646 }
1647 
1648 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1649 {
1650   return MachNode::size(ra_); // too many variables; just compute it
1651                               // the hard way
1652 }
1653 
1654 int MachPrologNode::reloc() const
1655 {
1656   return 0;
1657 }
1658 
1659 //=============================================================================
1660 
1661 #ifndef PRODUCT
1662 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1663   Compile* C = ra_->C;
1664   int framesize = C->frame_slots() << LogBytesPerInt;
1665 
1666   st->print("# pop frame %d\n\t",framesize);
1667 
1668   if (framesize == 0) {
1669     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1670   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1671     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1672     st->print("add  sp, sp, #%d\n\t", framesize);
1673   } else {
1674     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
1675     st->print("add  sp, sp, rscratch1\n\t");
1676     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1677   }
1678 
1679   if (do_polling() && C->is_method_compilation()) {
1680     st->print("# touch polling page\n\t");
1681     st->print("mov  rscratch1, #0x%lx\n\t", p2i(os::get_polling_page()));
1682     st->print("ldr zr, [rscratch1]");
1683   }
1684 }
1685 #endif
1686 
1687 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1688   Compile* C = ra_->C;
1689   MacroAssembler _masm(&cbuf);
1690   int framesize = C->frame_slots() << LogBytesPerInt;
1691 
1692   __ remove_frame(framesize);
1693 
1694   if (NotifySimulator) {
1695     __ notify(Assembler::method_reentry);
1696   }
1697 
1698   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1699     __ reserved_stack_check();
1700   }
1701 
1702   if (do_polling() && C->is_method_compilation()) {
1703     __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1704   }
1705 }
1706 
1707 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1708   // Variable size. Determine dynamically.
1709   return MachNode::size(ra_);
1710 }
1711 
1712 int MachEpilogNode::reloc() const {
1713   // Return number of relocatable values contained in this instruction.
1714   return 1; // 1 for polling page.
1715 }
1716 
1717 const Pipeline * MachEpilogNode::pipeline() const {
1718   return MachNode::pipeline_class();
1719 }
1720 
1721 // This method seems to be obsolete. It is declared in machnode.hpp
1722 // and defined in all *.ad files, but it is never called. Should we
1723 // get rid of it?
1724 int MachEpilogNode::safepoint_offset() const {
1725   assert(do_polling(), "no return for this epilog node");
1726   return 4;
1727 }
1728 
1729 //=============================================================================
1730 
1731 // Figure out which register class each belongs in: rc_int, rc_float or
1732 // rc_stack.
1733 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1734 
1735 static enum RC rc_class(OptoReg::Name reg) {
1736 
1737   if (reg == OptoReg::Bad) {
1738     return rc_bad;
1739   }
1740 
1741   // we have 30 int registers * 2 halves
1742   // (rscratch1 and rscratch2 are omitted)
1743 
1744   if (reg < 60) {
1745     return rc_int;
1746   }
1747 
1748   // we have 32 float register * 2 halves
1749   if (reg < 60 + 128) {
1750     return rc_float;
1751   }
1752 
1753   // Between float regs & stack is the flags regs.
1754   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1755 
1756   return rc_stack;
1757 }
1758 
1759 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1760   Compile* C = ra_->C;
1761 
1762   // Get registers to move.
1763   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1764   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1765   OptoReg::Name dst_hi = ra_->get_reg_second(this);
1766   OptoReg::Name dst_lo = ra_->get_reg_first(this);
1767 
1768   enum RC src_hi_rc = rc_class(src_hi);
1769   enum RC src_lo_rc = rc_class(src_lo);
1770   enum RC dst_hi_rc = rc_class(dst_hi);
1771   enum RC dst_lo_rc = rc_class(dst_lo);
1772 
1773   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1774 
1775   if (src_hi != OptoReg::Bad) {
1776     assert((src_lo&1)==0 && src_lo+1==src_hi &&
1777            (dst_lo&1)==0 && dst_lo+1==dst_hi,
1778            "expected aligned-adjacent pairs");
1779   }
1780 
1781   if (src_lo == dst_lo && src_hi == dst_hi) {
1782     return 0;            // Self copy, no move.
1783   }
1784 
1785   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1786               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1787   int src_offset = ra_->reg2offset(src_lo);
1788   int dst_offset = ra_->reg2offset(dst_lo);
1789 
1790   if (bottom_type()->isa_vect() != NULL) {
1791     uint ireg = ideal_reg();
1792     assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1793     if (cbuf) {
1794       MacroAssembler _masm(cbuf);
1795       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1796       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1797         // stack->stack
1798         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1799         if (ireg == Op_VecD) {
1800           __ unspill(rscratch1, true, src_offset);
1801           __ spill(rscratch1, true, dst_offset);
1802         } else {
1803           __ spill_copy128(src_offset, dst_offset);
1804         }
1805       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1806         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1807                ireg == Op_VecD ? __ T8B : __ T16B,
1808                as_FloatRegister(Matcher::_regEncode[src_lo]));
1809       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1810         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1811                        ireg == Op_VecD ? __ D : __ Q,
1812                        ra_->reg2offset(dst_lo));
1813       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1814         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1815                        ireg == Op_VecD ? __ D : __ Q,
1816                        ra_->reg2offset(src_lo));
1817       } else {
1818         ShouldNotReachHere();
1819       }
1820     }
1821   } else if (cbuf) {
1822     MacroAssembler _masm(cbuf);
1823     switch (src_lo_rc) {
1824     case rc_int:
1825       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
1826         if (is64) {
1827             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1828                    as_Register(Matcher::_regEncode[src_lo]));
1829         } else {
1830             MacroAssembler _masm(cbuf);
1831             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1832                     as_Register(Matcher::_regEncode[src_lo]));
1833         }
1834       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1835         if (is64) {
1836             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1837                      as_Register(Matcher::_regEncode[src_lo]));
1838         } else {
1839             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1840                      as_Register(Matcher::_regEncode[src_lo]));
1841         }
1842       } else {                    // gpr --> stack spill
1843         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1844         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1845       }
1846       break;
1847     case rc_float:
1848       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
1849         if (is64) {
1850             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1851                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1852         } else {
1853             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1854                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1855         }
1856       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1857           if (cbuf) {
1858             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1859                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1860         } else {
1861             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1862                      as_FloatRegister(Matcher::_regEncode[src_lo]));
1863         }
1864       } else {                    // fpr --> stack spill
1865         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1866         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1867                  is64 ? __ D : __ S, dst_offset);
1868       }
1869       break;
1870     case rc_stack:
1871       if (dst_lo_rc == rc_int) {  // stack --> gpr load
1872         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
1873       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
1874         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1875                    is64 ? __ D : __ S, src_offset);
1876       } else {                    // stack --> stack copy
1877         assert(dst_lo_rc == rc_stack, "spill to bad register class");
1878         __ unspill(rscratch1, is64, src_offset);
1879         __ spill(rscratch1, is64, dst_offset);
1880       }
1881       break;
1882     default:
1883       assert(false, "bad rc_class for spill");
1884       ShouldNotReachHere();
1885     }
1886   }
1887 
1888   if (st) {
1889     st->print("spill ");
1890     if (src_lo_rc == rc_stack) {
1891       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
1892     } else {
1893       st->print("%s -> ", Matcher::regName[src_lo]);
1894     }
1895     if (dst_lo_rc == rc_stack) {
1896       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
1897     } else {
1898       st->print("%s", Matcher::regName[dst_lo]);
1899     }
1900     if (bottom_type()->isa_vect() != NULL) {
1901       st->print("\t# vector spill size = %d", ideal_reg()==Op_VecD ? 64:128);
1902     } else {
1903       st->print("\t# spill size = %d", is64 ? 64:32);
1904     }
1905   }
1906 
1907   return 0;
1908 
1909 }
1910 
1911 #ifndef PRODUCT
1912 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1913   if (!ra_)
1914     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
1915   else
1916     implementation(NULL, ra_, false, st);
1917 }
1918 #endif
1919 
1920 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1921   implementation(&cbuf, ra_, false, NULL);
1922 }
1923 
1924 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1925   return MachNode::size(ra_);
1926 }
1927 
1928 //=============================================================================
1929 
1930 #ifndef PRODUCT
1931 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1932   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1933   int reg = ra_->get_reg_first(this);
1934   st->print("add %s, rsp, #%d]\t# box lock",
1935             Matcher::regName[reg], offset);
1936 }
1937 #endif
1938 
1939 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1940   MacroAssembler _masm(&cbuf);
1941 
1942   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1943   int reg    = ra_->get_encode(this);
1944 
1945   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
1946     __ add(as_Register(reg), sp, offset);
1947   } else {
1948     ShouldNotReachHere();
1949   }
1950 }
1951 
1952 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1953   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
1954   return 4;
1955 }
1956 
1957 //=============================================================================
1958 
1959 #ifndef PRODUCT
1960 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1961 {
1962   st->print_cr("# MachUEPNode");
1963   if (UseCompressedClassPointers) {
1964     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1965     if (CompressedKlassPointers::shift() != 0) {
1966       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1967     }
1968   } else {
1969    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1970   }
1971   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
1972   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
1973 }
1974 #endif
1975 
1976 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1977 {
1978   // This is the unverified entry point.
1979   MacroAssembler _masm(&cbuf);
1980 
1981   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
1982   Label skip;
1983   // TODO
1984   // can we avoid this skip and still use a reloc?
1985   __ br(Assembler::EQ, skip);
1986   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1987   __ bind(skip);
1988 }
1989 
1990 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1991 {
1992   return MachNode::size(ra_);
1993 }
1994 
1995 // REQUIRED EMIT CODE
1996 
1997 //=============================================================================
1998 
1999 // Emit exception handler code.
2000 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2001 {
2002   // mov rscratch1 #exception_blob_entry_point
2003   // br rscratch1
2004   // Note that the code buffer's insts_mark is always relative to insts.
2005   // That's why we must use the macroassembler to generate a handler.
2006   MacroAssembler _masm(&cbuf);
2007   address base = __ start_a_stub(size_exception_handler());
2008   if (base == NULL) {
2009     ciEnv::current()->record_failure("CodeCache is full");
2010     return 0;  // CodeBuffer::expand failed
2011   }
2012   int offset = __ offset();
2013   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2014   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2015   __ end_a_stub();
2016   return offset;
2017 }
2018 
2019 // Emit deopt handler code.
2020 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2021 {
2022   // Note that the code buffer's insts_mark is always relative to insts.
2023   // That's why we must use the macroassembler to generate a handler.
2024   MacroAssembler _masm(&cbuf);
2025   address base = __ start_a_stub(size_deopt_handler());
2026   if (base == NULL) {
2027     ciEnv::current()->record_failure("CodeCache is full");
2028     return 0;  // CodeBuffer::expand failed
2029   }
2030   int offset = __ offset();
2031 
2032   __ adr(lr, __ pc());
2033   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2034 
2035   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2036   __ end_a_stub();
2037   return offset;
2038 }
2039 
2040 // REQUIRED MATCHER CODE
2041 
2042 //=============================================================================
2043 
2044 const bool Matcher::match_rule_supported(int opcode) {
2045 
2046   switch (opcode) {
2047   default:
2048     break;
2049   }
2050 
2051   if (!has_match_rule(opcode)) {
2052     return false;
2053   }
2054 
2055   return true;  // Per default match rules are supported.
2056 }
2057 
2058 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
2059 
2060   // TODO
2061   // identify extra cases that we might want to provide match rules for
2062   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
2063   bool ret_value = match_rule_supported(opcode);
2064   // Add rules here.
2065 
2066   return ret_value;  // Per default match rules are supported.
2067 }
2068 
2069 const bool Matcher::has_predicated_vectors(void) {
2070   return false;
2071 }
2072 
2073 const int Matcher::float_pressure(int default_pressure_threshold) {
2074   return default_pressure_threshold;
2075 }
2076 
2077 int Matcher::regnum_to_fpu_offset(int regnum)
2078 {
2079   Unimplemented();
2080   return 0;
2081 }
2082 
2083 // Is this branch offset short enough that a short branch can be used?
2084 //
2085 // NOTE: If the platform does not provide any short branch variants, then
2086 //       this method should return false for offset 0.
2087 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2088   // The passed offset is relative to address of the branch.
2089 
2090   return (-32768 <= offset && offset < 32768);
2091 }
2092 
2093 const bool Matcher::isSimpleConstant64(jlong value) {
2094   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
2095   // Probably always true, even if a temp register is required.
2096   return true;
2097 }
2098 
2099 // true just means we have fast l2f conversion
2100 const bool Matcher::convL2FSupported(void) {
2101   return true;
2102 }
2103 
2104 // Vector width in bytes.
2105 const int Matcher::vector_width_in_bytes(BasicType bt) {
2106   int size = MIN2(16,(int)MaxVectorSize);
2107   // Minimum 2 values in vector
2108   if (size < 2*type2aelembytes(bt)) size = 0;
2109   // But never < 4
2110   if (size < 4) size = 0;
2111   return size;
2112 }
2113 
2114 // Limits on vector size (number of elements) loaded into vector.
2115 const int Matcher::max_vector_size(const BasicType bt) {
2116   return vector_width_in_bytes(bt)/type2aelembytes(bt);
2117 }
2118 const int Matcher::min_vector_size(const BasicType bt) {
2119 //  For the moment limit the vector size to 8 bytes
2120     int size = 8 / type2aelembytes(bt);
2121     if (size < 2) size = 2;
2122     return size;
2123 }
2124 
2125 // Vector ideal reg.
2126 const uint Matcher::vector_ideal_reg(int len) {
2127   switch(len) {
2128     case  8: return Op_VecD;
2129     case 16: return Op_VecX;
2130   }
2131   ShouldNotReachHere();
2132   return 0;
2133 }
2134 
2135 const uint Matcher::vector_shift_count_ideal_reg(int size) {
2136   switch(size) {
2137     case  8: return Op_VecD;
2138     case 16: return Op_VecX;
2139   }
2140   ShouldNotReachHere();
2141   return 0;
2142 }
2143 
2144 // AES support not yet implemented
2145 const bool Matcher::pass_original_key_for_aes() {
2146   return false;
2147 }
2148 
2149 // aarch64 supports misaligned vectors store/load.
2150 const bool Matcher::misaligned_vectors_ok() {
2151   return true;
2152 }
2153 
2154 // false => size gets scaled to BytesPerLong, ok.
2155 const bool Matcher::init_array_count_is_in_bytes = false;
2156 
2157 // Use conditional move (CMOVL)
2158 const int Matcher::long_cmove_cost() {
2159   // long cmoves are no more expensive than int cmoves
2160   return 0;
2161 }
2162 
2163 const int Matcher::float_cmove_cost() {
2164   // float cmoves are no more expensive than int cmoves
2165   return 0;
2166 }
2167 
2168 // Does the CPU require late expand (see block.cpp for description of late expand)?
2169 const bool Matcher::require_postalloc_expand = false;
2170 
2171 // Do we need to mask the count passed to shift instructions or does
2172 // the cpu only look at the lower 5/6 bits anyway?
2173 const bool Matcher::need_masked_shift_count = false;
2174 
2175 // This affects two different things:
2176 //  - how Decode nodes are matched
2177 //  - how ImplicitNullCheck opportunities are recognized
2178 // If true, the matcher will try to remove all Decodes and match them
2179 // (as operands) into nodes. NullChecks are not prepared to deal with
2180 // Decodes by final_graph_reshaping().
2181 // If false, final_graph_reshaping() forces the decode behind the Cmp
2182 // for a NullCheck. The matcher matches the Decode node into a register.
2183 // Implicit_null_check optimization moves the Decode along with the
2184 // memory operation back up before the NullCheck.
2185 bool Matcher::narrow_oop_use_complex_address() {
2186   return CompressedOops::shift() == 0;
2187 }
2188 
2189 bool Matcher::narrow_klass_use_complex_address() {
2190 // TODO
2191 // decide whether we need to set this to true
2192   return false;
2193 }
2194 
2195 bool Matcher::const_oop_prefer_decode() {
2196   // Prefer ConN+DecodeN over ConP in simple compressed oops mode.
2197   return CompressedOops::base() == NULL;
2198 }
2199 
2200 bool Matcher::const_klass_prefer_decode() {
2201   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
2202   return CompressedKlassPointers::base() == NULL;
2203 }
2204 
2205 // Is it better to copy float constants, or load them directly from
2206 // memory?  Intel can load a float constant from a direct address,
2207 // requiring no extra registers.  Most RISCs will have to materialize
2208 // an address into a register first, so they would do better to copy
2209 // the constant from stack.
2210 const bool Matcher::rematerialize_float_constants = false;
2211 
2212 // If CPU can load and store mis-aligned doubles directly then no
2213 // fixup is needed.  Else we split the double into 2 integer pieces
2214 // and move it piece-by-piece.  Only happens when passing doubles into
2215 // C code as the Java calling convention forces doubles to be aligned.
2216 const bool Matcher::misaligned_doubles_ok = true;
2217 
2218 // No-op on amd64
2219 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2220   Unimplemented();
2221 }
2222 
2223 // Advertise here if the CPU requires explicit rounding operations to
2224 // implement the UseStrictFP mode.
2225 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2226 
2227 // Are floats converted to double when stored to stack during
2228 // deoptimization?
2229 bool Matcher::float_in_double() { return false; }
2230 
2231 // Do ints take an entire long register or just half?
2232 // The relevant question is how the int is callee-saved:
2233 // the whole long is written but de-opt'ing will have to extract
2234 // the relevant 32 bits.
2235 const bool Matcher::int_in_long = true;
2236 
2237 // Return whether or not this register is ever used as an argument.
2238 // This function is used on startup to build the trampoline stubs in
2239 // generateOptoStub.  Registers not mentioned will be killed by the VM
2240 // call in the trampoline, and arguments in those registers not be
2241 // available to the callee.
2242 bool Matcher::can_be_java_arg(int reg)
2243 {
2244   return
2245     reg ==  R0_num || reg == R0_H_num ||
2246     reg ==  R1_num || reg == R1_H_num ||
2247     reg ==  R2_num || reg == R2_H_num ||
2248     reg ==  R3_num || reg == R3_H_num ||
2249     reg ==  R4_num || reg == R4_H_num ||
2250     reg ==  R5_num || reg == R5_H_num ||
2251     reg ==  R6_num || reg == R6_H_num ||
2252     reg ==  R7_num || reg == R7_H_num ||
2253     reg ==  V0_num || reg == V0_H_num ||
2254     reg ==  V1_num || reg == V1_H_num ||
2255     reg ==  V2_num || reg == V2_H_num ||
2256     reg ==  V3_num || reg == V3_H_num ||
2257     reg ==  V4_num || reg == V4_H_num ||
2258     reg ==  V5_num || reg == V5_H_num ||
2259     reg ==  V6_num || reg == V6_H_num ||
2260     reg ==  V7_num || reg == V7_H_num;
2261 }
2262 
2263 bool Matcher::is_spillable_arg(int reg)
2264 {
2265   return can_be_java_arg(reg);
2266 }
2267 
2268 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2269   return false;
2270 }
2271 
2272 RegMask Matcher::divI_proj_mask() {
2273   ShouldNotReachHere();
2274   return RegMask();
2275 }
2276 
2277 // Register for MODI projection of divmodI.
2278 RegMask Matcher::modI_proj_mask() {
2279   ShouldNotReachHere();
2280   return RegMask();
2281 }
2282 
2283 // Register for DIVL projection of divmodL.
2284 RegMask Matcher::divL_proj_mask() {
2285   ShouldNotReachHere();
2286   return RegMask();
2287 }
2288 
2289 // Register for MODL projection of divmodL.
2290 RegMask Matcher::modL_proj_mask() {
2291   ShouldNotReachHere();
2292   return RegMask();
2293 }
2294 
2295 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2296   return FP_REG_mask();
2297 }
2298 
2299 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2300   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2301     Node* u = addp->fast_out(i);
2302     if (u->is_Mem()) {
2303       int opsize = u->as_Mem()->memory_size();
2304       assert(opsize > 0, "unexpected memory operand size");
2305       if (u->as_Mem()->memory_size() != (1<<shift)) {
2306         return false;
2307       }
2308     }
2309   }
2310   return true;
2311 }
2312 
2313 const bool Matcher::convi2l_type_required = false;
2314 
2315 // Should the Matcher clone shifts on addressing modes, expecting them
2316 // to be subsumed into complex addressing expressions or compute them
2317 // into registers?
2318 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2319   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2320     return true;
2321   }
2322 
2323   Node *off = m->in(AddPNode::Offset);
2324   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2325       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2326       // Are there other uses besides address expressions?
2327       !is_visited(off)) {
2328     address_visited.set(off->_idx); // Flag as address_visited
2329     mstack.push(off->in(2), Visit);
2330     Node *conv = off->in(1);
2331     if (conv->Opcode() == Op_ConvI2L &&
2332         // Are there other uses besides address expressions?
2333         !is_visited(conv)) {
2334       address_visited.set(conv->_idx); // Flag as address_visited
2335       mstack.push(conv->in(1), Pre_Visit);
2336     } else {
2337       mstack.push(conv, Pre_Visit);
2338     }
2339     address_visited.test_set(m->_idx); // Flag as address_visited
2340     mstack.push(m->in(AddPNode::Address), Pre_Visit);
2341     mstack.push(m->in(AddPNode::Base), Pre_Visit);
2342     return true;
2343   } else if (off->Opcode() == Op_ConvI2L &&
2344              // Are there other uses besides address expressions?
2345              !is_visited(off)) {
2346     address_visited.test_set(m->_idx); // Flag as address_visited
2347     address_visited.set(off->_idx); // Flag as address_visited
2348     mstack.push(off->in(1), Pre_Visit);
2349     mstack.push(m->in(AddPNode::Address), Pre_Visit);
2350     mstack.push(m->in(AddPNode::Base), Pre_Visit);
2351     return true;
2352   }
2353   return false;
2354 }
2355 
2356 void Compile::reshape_address(AddPNode* addp) {
2357 }
2358 
2359 // helper for encoding java_to_runtime calls on sim
2360 //
2361 // this is needed to compute the extra arguments required when
2362 // planting a call to the simulator blrt instruction. the TypeFunc
2363 // can be queried to identify the counts for integral, and floating
2364 // arguments and the return type
2365 
2366 static void getCallInfo(const TypeFunc *tf, int &gpcnt, int &fpcnt, int &rtype)
2367 {
2368   int gps = 0;
2369   int fps = 0;
2370   const TypeTuple *domain = tf->domain();
2371   int max = domain->cnt();
2372   for (int i = TypeFunc::Parms; i < max; i++) {
2373     const Type *t = domain->field_at(i);
2374     switch(t->basic_type()) {
2375     case T_FLOAT:
2376     case T_DOUBLE:
2377       fps++;
2378     default:
2379       gps++;
2380     }
2381   }
2382   gpcnt = gps;
2383   fpcnt = fps;
2384   BasicType rt = tf->return_type();
2385   switch (rt) {
2386   case T_VOID:
2387     rtype = MacroAssembler::ret_type_void;
2388     break;
2389   default:
2390     rtype = MacroAssembler::ret_type_integral;
2391     break;
2392   case T_FLOAT:
2393     rtype = MacroAssembler::ret_type_float;
2394     break;
2395   case T_DOUBLE:
2396     rtype = MacroAssembler::ret_type_double;
2397     break;
2398   }
2399 }
2400 
2401 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
2402   MacroAssembler _masm(&cbuf);                                          \
2403   {                                                                     \
2404     guarantee(INDEX == -1, "mode not permitted for volatile");          \
2405     guarantee(DISP == 0, "mode not permitted for volatile");            \
2406     guarantee(SCALE == 0, "mode not permitted for volatile");           \
2407     __ INSN(REG, as_Register(BASE));                                    \
2408   }
2409 
2410 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2411 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2412 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2413                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
2414 
2415   // Used for all non-volatile memory accesses.  The use of
2416   // $mem->opcode() to discover whether this pattern uses sign-extended
2417   // offsets is something of a kludge.
2418   static void loadStore(MacroAssembler masm, mem_insn insn,
2419                          Register reg, int opcode,
2420                          Register base, int index, int size, int disp)
2421   {
2422     Address::extend scale;
2423 
2424     // Hooboy, this is fugly.  We need a way to communicate to the
2425     // encoder that the index needs to be sign extended, so we have to
2426     // enumerate all the cases.
2427     switch (opcode) {
2428     case INDINDEXSCALEDI2L:
2429     case INDINDEXSCALEDI2LN:
2430     case INDINDEXI2L:
2431     case INDINDEXI2LN:
2432       scale = Address::sxtw(size);
2433       break;
2434     default:
2435       scale = Address::lsl(size);
2436     }
2437 
2438     if (index == -1) {
2439       (masm.*insn)(reg, Address(base, disp));
2440     } else {
2441       assert(disp == 0, "unsupported address mode: disp = %d", disp);
2442       (masm.*insn)(reg, Address(base, as_Register(index), scale));
2443     }
2444   }
2445 
2446   static void loadStore(MacroAssembler masm, mem_float_insn insn,
2447                          FloatRegister reg, int opcode,
2448                          Register base, int index, int size, int disp)
2449   {
2450     Address::extend scale;
2451 
2452     switch (opcode) {
2453     case INDINDEXSCALEDI2L:
2454     case INDINDEXSCALEDI2LN:
2455       scale = Address::sxtw(size);
2456       break;
2457     default:
2458       scale = Address::lsl(size);
2459     }
2460 
2461      if (index == -1) {
2462       (masm.*insn)(reg, Address(base, disp));
2463     } else {
2464       assert(disp == 0, "unsupported address mode: disp = %d", disp);
2465       (masm.*insn)(reg, Address(base, as_Register(index), scale));
2466     }
2467   }
2468 
2469   static void loadStore(MacroAssembler masm, mem_vector_insn insn,
2470                          FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2471                          int opcode, Register base, int index, int size, int disp)
2472   {
2473     if (index == -1) {
2474       (masm.*insn)(reg, T, Address(base, disp));
2475     } else {
2476       assert(disp == 0, "unsupported address mode");
2477       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2478     }
2479   }
2480 
2481 %}
2482 
2483 
2484 
2485 //----------ENCODING BLOCK-----------------------------------------------------
2486 // This block specifies the encoding classes used by the compiler to
2487 // output byte streams.  Encoding classes are parameterized macros
2488 // used by Machine Instruction Nodes in order to generate the bit
2489 // encoding of the instruction.  Operands specify their base encoding
2490 // interface with the interface keyword.  There are currently
2491 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2492 // COND_INTER.  REG_INTER causes an operand to generate a function
2493 // which returns its register number when queried.  CONST_INTER causes
2494 // an operand to generate a function which returns the value of the
2495 // constant when queried.  MEMORY_INTER causes an operand to generate
2496 // four functions which return the Base Register, the Index Register,
2497 // the Scale Value, and the Offset Value of the operand when queried.
2498 // COND_INTER causes an operand to generate six functions which return
2499 // the encoding code (ie - encoding bits for the instruction)
2500 // associated with each basic boolean condition for a conditional
2501 // instruction.
2502 //
2503 // Instructions specify two basic values for encoding.  Again, a
2504 // function is available to check if the constant displacement is an
2505 // oop. They use the ins_encode keyword to specify their encoding
2506 // classes (which must be a sequence of enc_class names, and their
2507 // parameters, specified in the encoding block), and they use the
2508 // opcode keyword to specify, in order, their primary, secondary, and
2509 // tertiary opcode.  Only the opcode sections which a particular
2510 // instruction needs for encoding need to be specified.
2511 encode %{
2512   // Build emit functions for each basic byte or larger field in the
2513   // intel encoding scheme (opcode, rm, sib, immediate), and call them
2514   // from C++ code in the enc_class source block.  Emit functions will
2515   // live in the main source block for now.  In future, we can
2516   // generalize this by adding a syntax that specifies the sizes of
2517   // fields in an order, so that the adlc can build the emit functions
2518   // automagically
2519 
2520   // catch all for unimplemented encodings
2521   enc_class enc_unimplemented %{
2522     MacroAssembler _masm(&cbuf);
2523     __ unimplemented("C2 catch all");
2524   %}
2525 
2526   // BEGIN Non-volatile memory access
2527 
2528   enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
2529     Register dst_reg = as_Register($dst$$reg);
2530     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2531                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2532   %}
2533 
2534   enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
2535     Register dst_reg = as_Register($dst$$reg);
2536     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2537                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2538   %}
2539 
2540   enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
2541     Register dst_reg = as_Register($dst$$reg);
2542     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2543                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2544   %}
2545 
2546   enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
2547     Register dst_reg = as_Register($dst$$reg);
2548     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2549                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2550   %}
2551 
2552   enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
2553     Register dst_reg = as_Register($dst$$reg);
2554     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2555                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2556   %}
2557 
2558   enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
2559     Register dst_reg = as_Register($dst$$reg);
2560     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2561                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2562   %}
2563 
2564   enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
2565     Register dst_reg = as_Register($dst$$reg);
2566     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2567                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2568   %}
2569 
2570   enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
2571     Register dst_reg = as_Register($dst$$reg);
2572     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2573                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2574   %}
2575 
2576   enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
2577     Register dst_reg = as_Register($dst$$reg);
2578     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2579                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2580   %}
2581 
2582   enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
2583     Register dst_reg = as_Register($dst$$reg);
2584     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2585                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2586   %}
2587 
2588   enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
2589     Register dst_reg = as_Register($dst$$reg);
2590     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2591                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2592   %}
2593 
2594   enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
2595     Register dst_reg = as_Register($dst$$reg);
2596     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2597                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2598   %}
2599 
2600   enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
2601     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2602     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2603                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2604   %}
2605 
2606   enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
2607     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2608     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2609                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2610   %}
2611 
2612   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
2613     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2614     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2615        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2616   %}
2617 
2618   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
2619     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2620     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
2621        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2622   %}
2623 
2624   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
2625     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2626     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
2627        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2628   %}
2629 
2630   enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2631     Register src_reg = as_Register($src$$reg);
2632     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2633                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2634   %}
2635 
2636   enc_class aarch64_enc_strb0(memory mem) %{
2637     MacroAssembler _masm(&cbuf);
2638     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2639                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2640   %}
2641 
2642   enc_class aarch64_enc_strb0_ordered(memory mem) %{
2643     MacroAssembler _masm(&cbuf);
2644     __ membar(Assembler::StoreStore);
2645     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2646                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2647   %}
2648 
2649   enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2650     Register src_reg = as_Register($src$$reg);
2651     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2652                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2653   %}
2654 
2655   enc_class aarch64_enc_strh0(memory mem) %{
2656     MacroAssembler _masm(&cbuf);
2657     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2658                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2659   %}
2660 
2661   enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2662     Register src_reg = as_Register($src$$reg);
2663     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2664                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2665   %}
2666 
2667   enc_class aarch64_enc_strw0(memory mem) %{
2668     MacroAssembler _masm(&cbuf);
2669     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2670                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2671   %}
2672 
2673   enc_class aarch64_enc_str(iRegL src, memory mem) %{
2674     Register src_reg = as_Register($src$$reg);
2675     // we sometimes get asked to store the stack pointer into the
2676     // current thread -- we cannot do that directly on AArch64
2677     if (src_reg == r31_sp) {
2678       MacroAssembler _masm(&cbuf);
2679       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2680       __ mov(rscratch2, sp);
2681       src_reg = rscratch2;
2682     }
2683     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2684                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2685   %}
2686 
2687   enc_class aarch64_enc_str0(memory mem) %{
2688     MacroAssembler _masm(&cbuf);
2689     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2690                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2691   %}
2692 
2693   enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2694     FloatRegister src_reg = as_FloatRegister($src$$reg);
2695     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2696                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2697   %}
2698 
2699   enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2700     FloatRegister src_reg = as_FloatRegister($src$$reg);
2701     loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2702                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2703   %}
2704 
2705   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
2706     FloatRegister src_reg = as_FloatRegister($src$$reg);
2707     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
2708        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2709   %}
2710 
2711   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
2712     FloatRegister src_reg = as_FloatRegister($src$$reg);
2713     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
2714        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2715   %}
2716 
2717   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
2718     FloatRegister src_reg = as_FloatRegister($src$$reg);
2719     loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
2720        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2721   %}
2722 
2723   // END Non-volatile memory access
2724 
2725   // volatile loads and stores
2726 
2727   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2728     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2729                  rscratch1, stlrb);
2730   %}
2731 
2732   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2733     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2734                  rscratch1, stlrh);
2735   %}
2736 
2737   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2738     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2739                  rscratch1, stlrw);
2740   %}
2741 
2742 
2743   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2744     Register dst_reg = as_Register($dst$$reg);
2745     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2746              rscratch1, ldarb);
2747     __ sxtbw(dst_reg, dst_reg);
2748   %}
2749 
2750   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2751     Register dst_reg = as_Register($dst$$reg);
2752     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2753              rscratch1, ldarb);
2754     __ sxtb(dst_reg, dst_reg);
2755   %}
2756 
2757   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2758     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2759              rscratch1, ldarb);
2760   %}
2761 
2762   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2763     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2764              rscratch1, ldarb);
2765   %}
2766 
2767   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2768     Register dst_reg = as_Register($dst$$reg);
2769     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2770              rscratch1, ldarh);
2771     __ sxthw(dst_reg, dst_reg);
2772   %}
2773 
2774   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2775     Register dst_reg = as_Register($dst$$reg);
2776     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2777              rscratch1, ldarh);
2778     __ sxth(dst_reg, dst_reg);
2779   %}
2780 
2781   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2782     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2783              rscratch1, ldarh);
2784   %}
2785 
2786   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2787     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2788              rscratch1, ldarh);
2789   %}
2790 
2791   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2792     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2793              rscratch1, ldarw);
2794   %}
2795 
2796   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2797     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2798              rscratch1, ldarw);
2799   %}
2800 
2801   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2802     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2803              rscratch1, ldar);
2804   %}
2805 
2806   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2807     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2808              rscratch1, ldarw);
2809     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2810   %}
2811 
2812   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2813     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2814              rscratch1, ldar);
2815     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2816   %}
2817 
2818   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2819     Register src_reg = as_Register($src$$reg);
2820     // we sometimes get asked to store the stack pointer into the
2821     // current thread -- we cannot do that directly on AArch64
2822     if (src_reg == r31_sp) {
2823         MacroAssembler _masm(&cbuf);
2824       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2825       __ mov(rscratch2, sp);
2826       src_reg = rscratch2;
2827     }
2828     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2829                  rscratch1, stlr);
2830   %}
2831 
2832   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2833     {
2834       MacroAssembler _masm(&cbuf);
2835       FloatRegister src_reg = as_FloatRegister($src$$reg);
2836       __ fmovs(rscratch2, src_reg);
2837     }
2838     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2839                  rscratch1, stlrw);
2840   %}
2841 
2842   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2843     {
2844       MacroAssembler _masm(&cbuf);
2845       FloatRegister src_reg = as_FloatRegister($src$$reg);
2846       __ fmovd(rscratch2, src_reg);
2847     }
2848     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2849                  rscratch1, stlr);
2850   %}
2851 
2852   // synchronized read/update encodings
2853 
2854   enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
2855     MacroAssembler _masm(&cbuf);
2856     Register dst_reg = as_Register($dst$$reg);
2857     Register base = as_Register($mem$$base);
2858     int index = $mem$$index;
2859     int scale = $mem$$scale;
2860     int disp = $mem$$disp;
2861     if (index == -1) {
2862        if (disp != 0) {
2863         __ lea(rscratch1, Address(base, disp));
2864         __ ldaxr(dst_reg, rscratch1);
2865       } else {
2866         // TODO
2867         // should we ever get anything other than this case?
2868         __ ldaxr(dst_reg, base);
2869       }
2870     } else {
2871       Register index_reg = as_Register(index);
2872       if (disp == 0) {
2873         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
2874         __ ldaxr(dst_reg, rscratch1);
2875       } else {
2876         __ lea(rscratch1, Address(base, disp));
2877         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
2878         __ ldaxr(dst_reg, rscratch1);
2879       }
2880     }
2881   %}
2882 
2883   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
2884     MacroAssembler _masm(&cbuf);
2885     Register src_reg = as_Register($src$$reg);
2886     Register base = as_Register($mem$$base);
2887     int index = $mem$$index;
2888     int scale = $mem$$scale;
2889     int disp = $mem$$disp;
2890     if (index == -1) {
2891        if (disp != 0) {
2892         __ lea(rscratch2, Address(base, disp));
2893         __ stlxr(rscratch1, src_reg, rscratch2);
2894       } else {
2895         // TODO
2896         // should we ever get anything other than this case?
2897         __ stlxr(rscratch1, src_reg, base);
2898       }
2899     } else {
2900       Register index_reg = as_Register(index);
2901       if (disp == 0) {
2902         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2903         __ stlxr(rscratch1, src_reg, rscratch2);
2904       } else {
2905         __ lea(rscratch2, Address(base, disp));
2906         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2907         __ stlxr(rscratch1, src_reg, rscratch2);
2908       }
2909     }
2910     __ cmpw(rscratch1, zr);
2911   %}
2912 
2913   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2914     MacroAssembler _masm(&cbuf);
2915     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2916     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2917                Assembler::xword, /*acquire*/ false, /*release*/ true,
2918                /*weak*/ false, noreg);
2919   %}
2920 
2921   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2922     MacroAssembler _masm(&cbuf);
2923     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2924     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2925                Assembler::word, /*acquire*/ false, /*release*/ true,
2926                /*weak*/ false, noreg);
2927   %}
2928 
2929   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2930     MacroAssembler _masm(&cbuf);
2931     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2932     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2933                Assembler::halfword, /*acquire*/ false, /*release*/ true,
2934                /*weak*/ false, noreg);
2935   %}
2936 
2937   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2938     MacroAssembler _masm(&cbuf);
2939     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2940     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2941                Assembler::byte, /*acquire*/ false, /*release*/ true,
2942                /*weak*/ false, noreg);
2943   %}
2944 
2945 
2946   // The only difference between aarch64_enc_cmpxchg and
2947   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
2948   // CompareAndSwap sequence to serve as a barrier on acquiring a
2949   // lock.
2950   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2951     MacroAssembler _masm(&cbuf);
2952     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2953     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2954                Assembler::xword, /*acquire*/ true, /*release*/ true,
2955                /*weak*/ false, noreg);
2956   %}
2957 
2958   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2959     MacroAssembler _masm(&cbuf);
2960     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2961     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2962                Assembler::word, /*acquire*/ true, /*release*/ true,
2963                /*weak*/ false, noreg);
2964   %}
2965 
2966   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2967     MacroAssembler _masm(&cbuf);
2968     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2969     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2970                Assembler::halfword, /*acquire*/ true, /*release*/ true,
2971                /*weak*/ false, noreg);
2972   %}
2973 
2974   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2975     MacroAssembler _masm(&cbuf);
2976     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2977     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2978                Assembler::byte, /*acquire*/ true, /*release*/ true,
2979                /*weak*/ false, noreg);
2980   %}
2981 
2982   // auxiliary used for CompareAndSwapX to set result register
2983   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
2984     MacroAssembler _masm(&cbuf);
2985     Register res_reg = as_Register($res$$reg);
2986     __ cset(res_reg, Assembler::EQ);
2987   %}
2988 
2989   // prefetch encodings
2990 
2991   enc_class aarch64_enc_prefetchw(memory mem) %{
2992     MacroAssembler _masm(&cbuf);
2993     Register base = as_Register($mem$$base);
2994     int index = $mem$$index;
2995     int scale = $mem$$scale;
2996     int disp = $mem$$disp;
2997     if (index == -1) {
2998       __ prfm(Address(base, disp), PSTL1KEEP);
2999     } else {
3000       Register index_reg = as_Register(index);
3001       if (disp == 0) {
3002         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3003       } else {
3004         __ lea(rscratch1, Address(base, disp));
3005         __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3006       }
3007     }
3008   %}
3009 
3010   /// mov envcodings
3011 
3012   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3013     MacroAssembler _masm(&cbuf);
3014     u_int32_t con = (u_int32_t)$src$$constant;
3015     Register dst_reg = as_Register($dst$$reg);
3016     if (con == 0) {
3017       __ movw(dst_reg, zr);
3018     } else {
3019       __ movw(dst_reg, con);
3020     }
3021   %}
3022 
3023   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3024     MacroAssembler _masm(&cbuf);
3025     Register dst_reg = as_Register($dst$$reg);
3026     u_int64_t con = (u_int64_t)$src$$constant;
3027     if (con == 0) {
3028       __ mov(dst_reg, zr);
3029     } else {
3030       __ mov(dst_reg, con);
3031     }
3032   %}
3033 
3034   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3035     MacroAssembler _masm(&cbuf);
3036     Register dst_reg = as_Register($dst$$reg);
3037     address con = (address)$src$$constant;
3038     if (con == NULL || con == (address)1) {
3039       ShouldNotReachHere();
3040     } else {
3041       relocInfo::relocType rtype = $src->constant_reloc();
3042       if (rtype == relocInfo::oop_type) {
3043         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3044       } else if (rtype == relocInfo::metadata_type) {
3045         __ mov_metadata(dst_reg, (Metadata*)con);
3046       } else {
3047         assert(rtype == relocInfo::none, "unexpected reloc type");
3048         if (con < (address)(uintptr_t)os::vm_page_size()) {
3049           __ mov(dst_reg, con);
3050         } else {
3051           unsigned long offset;
3052           __ adrp(dst_reg, con, offset);
3053           __ add(dst_reg, dst_reg, offset);
3054         }
3055       }
3056     }
3057   %}
3058 
3059   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3060     MacroAssembler _masm(&cbuf);
3061     Register dst_reg = as_Register($dst$$reg);
3062     __ mov(dst_reg, zr);
3063   %}
3064 
3065   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3066     MacroAssembler _masm(&cbuf);
3067     Register dst_reg = as_Register($dst$$reg);
3068     __ mov(dst_reg, (u_int64_t)1);
3069   %}
3070 
3071   enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
3072     MacroAssembler _masm(&cbuf);
3073     address page = (address)$src$$constant;
3074     Register dst_reg = as_Register($dst$$reg);
3075     unsigned long off;
3076     __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
3077     assert(off == 0, "assumed offset == 0");
3078   %}
3079 
3080   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3081     MacroAssembler _masm(&cbuf);
3082     __ load_byte_map_base($dst$$Register);
3083   %}
3084 
3085   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3086     MacroAssembler _masm(&cbuf);
3087     Register dst_reg = as_Register($dst$$reg);
3088     address con = (address)$src$$constant;
3089     if (con == NULL) {
3090       ShouldNotReachHere();
3091     } else {
3092       relocInfo::relocType rtype = $src->constant_reloc();
3093       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3094       __ set_narrow_oop(dst_reg, (jobject)con);
3095     }
3096   %}
3097 
3098   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3099     MacroAssembler _masm(&cbuf);
3100     Register dst_reg = as_Register($dst$$reg);
3101     __ mov(dst_reg, zr);
3102   %}
3103 
3104   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3105     MacroAssembler _masm(&cbuf);
3106     Register dst_reg = as_Register($dst$$reg);
3107     address con = (address)$src$$constant;
3108     if (con == NULL) {
3109       ShouldNotReachHere();
3110     } else {
3111       relocInfo::relocType rtype = $src->constant_reloc();
3112       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3113       __ set_narrow_klass(dst_reg, (Klass *)con);
3114     }
3115   %}
3116 
3117   // arithmetic encodings
3118 
3119   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3120     MacroAssembler _masm(&cbuf);
3121     Register dst_reg = as_Register($dst$$reg);
3122     Register src_reg = as_Register($src1$$reg);
3123     int32_t con = (int32_t)$src2$$constant;
3124     // add has primary == 0, subtract has primary == 1
3125     if ($primary) { con = -con; }
3126     if (con < 0) {
3127       __ subw(dst_reg, src_reg, -con);
3128     } else {
3129       __ addw(dst_reg, src_reg, con);
3130     }
3131   %}
3132 
3133   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3134     MacroAssembler _masm(&cbuf);
3135     Register dst_reg = as_Register($dst$$reg);
3136     Register src_reg = as_Register($src1$$reg);
3137     int32_t con = (int32_t)$src2$$constant;
3138     // add has primary == 0, subtract has primary == 1
3139     if ($primary) { con = -con; }
3140     if (con < 0) {
3141       __ sub(dst_reg, src_reg, -con);
3142     } else {
3143       __ add(dst_reg, src_reg, con);
3144     }
3145   %}
3146 
3147   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3148     MacroAssembler _masm(&cbuf);
3149    Register dst_reg = as_Register($dst$$reg);
3150    Register src1_reg = as_Register($src1$$reg);
3151    Register src2_reg = as_Register($src2$$reg);
3152     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3153   %}
3154 
3155   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3156     MacroAssembler _masm(&cbuf);
3157    Register dst_reg = as_Register($dst$$reg);
3158    Register src1_reg = as_Register($src1$$reg);
3159    Register src2_reg = as_Register($src2$$reg);
3160     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3161   %}
3162 
3163   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3164     MacroAssembler _masm(&cbuf);
3165    Register dst_reg = as_Register($dst$$reg);
3166    Register src1_reg = as_Register($src1$$reg);
3167    Register src2_reg = as_Register($src2$$reg);
3168     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3169   %}
3170 
3171   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3172     MacroAssembler _masm(&cbuf);
3173    Register dst_reg = as_Register($dst$$reg);
3174    Register src1_reg = as_Register($src1$$reg);
3175    Register src2_reg = as_Register($src2$$reg);
3176     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3177   %}
3178 
3179   // compare instruction encodings
3180 
3181   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3182     MacroAssembler _masm(&cbuf);
3183     Register reg1 = as_Register($src1$$reg);
3184     Register reg2 = as_Register($src2$$reg);
3185     __ cmpw(reg1, reg2);
3186   %}
3187 
3188   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3189     MacroAssembler _masm(&cbuf);
3190     Register reg = as_Register($src1$$reg);
3191     int32_t val = $src2$$constant;
3192     if (val >= 0) {
3193       __ subsw(zr, reg, val);
3194     } else {
3195       __ addsw(zr, reg, -val);
3196     }
3197   %}
3198 
3199   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3200     MacroAssembler _masm(&cbuf);
3201     Register reg1 = as_Register($src1$$reg);
3202     u_int32_t val = (u_int32_t)$src2$$constant;
3203     __ movw(rscratch1, val);
3204     __ cmpw(reg1, rscratch1);
3205   %}
3206 
3207   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3208     MacroAssembler _masm(&cbuf);
3209     Register reg1 = as_Register($src1$$reg);
3210     Register reg2 = as_Register($src2$$reg);
3211     __ cmp(reg1, reg2);
3212   %}
3213 
3214   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3215     MacroAssembler _masm(&cbuf);
3216     Register reg = as_Register($src1$$reg);
3217     int64_t val = $src2$$constant;
3218     if (val >= 0) {
3219       __ subs(zr, reg, val);
3220     } else if (val != -val) {
3221       __ adds(zr, reg, -val);
3222     } else {
3223     // aargh, Long.MIN_VALUE is a special case
3224       __ orr(rscratch1, zr, (u_int64_t)val);
3225       __ subs(zr, reg, rscratch1);
3226     }
3227   %}
3228 
3229   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3230     MacroAssembler _masm(&cbuf);
3231     Register reg1 = as_Register($src1$$reg);
3232     u_int64_t val = (u_int64_t)$src2$$constant;
3233     __ mov(rscratch1, val);
3234     __ cmp(reg1, rscratch1);
3235   %}
3236 
3237   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3238     MacroAssembler _masm(&cbuf);
3239     Register reg1 = as_Register($src1$$reg);
3240     Register reg2 = as_Register($src2$$reg);
3241     __ cmp(reg1, reg2);
3242   %}
3243 
3244   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3245     MacroAssembler _masm(&cbuf);
3246     Register reg1 = as_Register($src1$$reg);
3247     Register reg2 = as_Register($src2$$reg);
3248     __ cmpw(reg1, reg2);
3249   %}
3250 
3251   enc_class aarch64_enc_testp(iRegP src) %{
3252     MacroAssembler _masm(&cbuf);
3253     Register reg = as_Register($src$$reg);
3254     __ cmp(reg, zr);
3255   %}
3256 
3257   enc_class aarch64_enc_testn(iRegN src) %{
3258     MacroAssembler _masm(&cbuf);
3259     Register reg = as_Register($src$$reg);
3260     __ cmpw(reg, zr);
3261   %}
3262 
3263   enc_class aarch64_enc_b(label lbl) %{
3264     MacroAssembler _masm(&cbuf);
3265     Label *L = $lbl$$label;
3266     __ b(*L);
3267   %}
3268 
3269   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3270     MacroAssembler _masm(&cbuf);
3271     Label *L = $lbl$$label;
3272     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3273   %}
3274 
3275   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3276     MacroAssembler _masm(&cbuf);
3277     Label *L = $lbl$$label;
3278     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3279   %}
3280 
3281   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3282   %{
3283      Register sub_reg = as_Register($sub$$reg);
3284      Register super_reg = as_Register($super$$reg);
3285      Register temp_reg = as_Register($temp$$reg);
3286      Register result_reg = as_Register($result$$reg);
3287 
3288      Label miss;
3289      MacroAssembler _masm(&cbuf);
3290      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3291                                      NULL, &miss,
3292                                      /*set_cond_codes:*/ true);
3293      if ($primary) {
3294        __ mov(result_reg, zr);
3295      }
3296      __ bind(miss);
3297   %}
3298 
3299   enc_class aarch64_enc_java_static_call(method meth) %{
3300     MacroAssembler _masm(&cbuf);
3301 
3302     address addr = (address)$meth$$method;
3303     address call;
3304     if (!_method) {
3305       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3306       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3307     } else {
3308       int method_index = resolved_method_index(cbuf);
3309       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3310                                                   : static_call_Relocation::spec(method_index);
3311       call = __ trampoline_call(Address(addr, rspec), &cbuf);
3312 
3313       // Emit stub for static call
3314       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3315       if (stub == NULL) {
3316         ciEnv::current()->record_failure("CodeCache is full");
3317         return;
3318       }
3319     }
3320     if (call == NULL) {
3321       ciEnv::current()->record_failure("CodeCache is full");
3322       return;
3323     }
3324   %}
3325 
3326   enc_class aarch64_enc_java_dynamic_call(method meth) %{
3327     MacroAssembler _masm(&cbuf);
3328     int method_index = resolved_method_index(cbuf);
3329     address call = __ ic_call((address)$meth$$method, method_index);
3330     if (call == NULL) {
3331       ciEnv::current()->record_failure("CodeCache is full");
3332       return;
3333     }
3334   %}
3335 
3336   enc_class aarch64_enc_call_epilog() %{
3337     MacroAssembler _masm(&cbuf);
3338     if (VerifyStackAtCalls) {
3339       // Check that stack depth is unchanged: find majik cookie on stack
3340       __ call_Unimplemented();
3341     }
3342   %}
3343 
3344   enc_class aarch64_enc_java_to_runtime(method meth) %{
3345     MacroAssembler _masm(&cbuf);
3346 
3347     // some calls to generated routines (arraycopy code) are scheduled
3348     // by C2 as runtime calls. if so we can call them using a br (they
3349     // will be in a reachable segment) otherwise we have to use a blrt
3350     // which loads the absolute address into a register.
3351     address entry = (address)$meth$$method;
3352     CodeBlob *cb = CodeCache::find_blob(entry);
3353     if (cb) {
3354       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3355       if (call == NULL) {
3356         ciEnv::current()->record_failure("CodeCache is full");
3357         return;
3358       }
3359     } else {
3360       int gpcnt;
3361       int fpcnt;
3362       int rtype;
3363       getCallInfo(tf(), gpcnt, fpcnt, rtype);
3364       Label retaddr;
3365       __ adr(rscratch2, retaddr);
3366       __ lea(rscratch1, RuntimeAddress(entry));
3367       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3368       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3369       __ blrt(rscratch1, gpcnt, fpcnt, rtype);
3370       __ bind(retaddr);
3371       __ add(sp, sp, 2 * wordSize);
3372     }
3373   %}
3374 
3375   enc_class aarch64_enc_rethrow() %{
3376     MacroAssembler _masm(&cbuf);
3377     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3378   %}
3379 
3380   enc_class aarch64_enc_ret() %{
3381     MacroAssembler _masm(&cbuf);
3382     __ ret(lr);
3383   %}
3384 
3385   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3386     MacroAssembler _masm(&cbuf);
3387     Register target_reg = as_Register($jump_target$$reg);
3388     __ br(target_reg);
3389   %}
3390 
3391   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3392     MacroAssembler _masm(&cbuf);
3393     Register target_reg = as_Register($jump_target$$reg);
3394     // exception oop should be in r0
3395     // ret addr has been popped into lr
3396     // callee expects it in r3
3397     __ mov(r3, lr);
3398     __ br(target_reg);
3399   %}
3400 
3401   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3402     MacroAssembler _masm(&cbuf);
3403     Register oop = as_Register($object$$reg);
3404     Register box = as_Register($box$$reg);
3405     Register disp_hdr = as_Register($tmp$$reg);
3406     Register tmp = as_Register($tmp2$$reg);
3407     Label cont;
3408     Label object_has_monitor;
3409     Label cas_failed;
3410 
3411     assert_different_registers(oop, box, tmp, disp_hdr);
3412 
3413     // Load markOop from object into displaced_header.
3414     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3415 
3416     if (UseBiasedLocking && !UseOptoBiasInlining) {
3417       __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3418     }
3419 
3420     // Check for existing monitor
3421     __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3422 
3423     // Set tmp to be (markOop of object | UNLOCK_VALUE).
3424     __ orr(tmp, disp_hdr, markOopDesc::unlocked_value);
3425 
3426     // Initialize the box. (Must happen before we update the object mark!)
3427     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3428 
3429     // Compare object markOop with an unlocked value (tmp) and if
3430     // equal exchange the stack address of our box with object markOop.
3431     // On failure disp_hdr contains the possibly locked markOop.
3432     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3433                /*release*/ true, /*weak*/ false, disp_hdr);
3434     __ br(Assembler::EQ, cont);
3435 
3436     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3437 
3438     // If the compare-and-exchange succeeded, then we found an unlocked
3439     // object, will have now locked it will continue at label cont
3440 
3441     __ bind(cas_failed);
3442     // We did not see an unlocked object so try the fast recursive case.
3443 
3444     // Check if the owner is self by comparing the value in the
3445     // markOop of object (disp_hdr) with the stack pointer.
3446     __ mov(rscratch1, sp);
3447     __ sub(disp_hdr, disp_hdr, rscratch1);
3448     __ mov(tmp, (address) (~(os::vm_page_size()-1) | (uintptr_t)markOopDesc::lock_mask_in_place));
3449     // If condition is true we are cont and hence we can store 0 as the
3450     // displaced header in the box, which indicates that it is a recursive lock.
3451     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
3452     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3453 
3454     __ b(cont);
3455 
3456     // Handle existing monitor.
3457     __ bind(object_has_monitor);
3458 
3459     // The object's monitor m is unlocked iff m->owner == NULL,
3460     // otherwise m->owner may contain a thread or a stack address.
3461     //
3462     // Try to CAS m->owner from NULL to current thread.
3463     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
3464     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3465                /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
3466 
3467     // Store a non-null value into the box to avoid looking like a re-entrant
3468     // lock. The fast-path monitor unlock code checks for
3469     // markOopDesc::monitor_value so use markOopDesc::unused_mark which has the
3470     // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
3471     __ mov(tmp, (address)markOopDesc::unused_mark());
3472     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3473 
3474     __ bind(cont);
3475     // flag == EQ indicates success
3476     // flag == NE indicates failure
3477   %}
3478 
3479   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3480     MacroAssembler _masm(&cbuf);
3481     Register oop = as_Register($object$$reg);
3482     Register box = as_Register($box$$reg);
3483     Register disp_hdr = as_Register($tmp$$reg);
3484     Register tmp = as_Register($tmp2$$reg);
3485     Label cont;
3486     Label object_has_monitor;
3487 
3488     assert_different_registers(oop, box, tmp, disp_hdr);
3489 
3490     if (UseBiasedLocking && !UseOptoBiasInlining) {
3491       __ biased_locking_exit(oop, tmp, cont);
3492     }
3493 
3494     // Find the lock address and load the displaced header from the stack.
3495     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3496 
3497     // If the displaced header is 0, we have a recursive unlock.
3498     __ cmp(disp_hdr, zr);
3499     __ br(Assembler::EQ, cont);
3500 
3501     // Handle existing monitor.
3502     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3503     __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3504 
3505     // Check if it is still a light weight lock, this is is true if we
3506     // see the stack address of the basicLock in the markOop of the
3507     // object.
3508 
3509     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3510                /*release*/ true, /*weak*/ false, tmp);
3511     __ b(cont);
3512 
3513     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3514 
3515     // Handle existing monitor.
3516     __ bind(object_has_monitor);
3517     __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3518     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3519     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3520     __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3521     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3522     __ cmp(rscratch1, zr); // Sets flags for result
3523     __ br(Assembler::NE, cont);
3524 
3525     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3526     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3527     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3528     __ cmp(rscratch1, zr); // Sets flags for result
3529     __ cbnz(rscratch1, cont);
3530     // need a release store here
3531     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3532     __ stlr(zr, tmp); // set unowned
3533 
3534     __ bind(cont);
3535     // flag == EQ indicates success
3536     // flag == NE indicates failure
3537   %}
3538 
3539 %}
3540 
3541 //----------FRAME--------------------------------------------------------------
3542 // Definition of frame structure and management information.
3543 //
3544 //  S T A C K   L A Y O U T    Allocators stack-slot number
3545 //                             |   (to get allocators register number
3546 //  G  Owned by    |        |  v    add OptoReg::stack0())
3547 //  r   CALLER     |        |
3548 //  o     |        +--------+      pad to even-align allocators stack-slot
3549 //  w     V        |  pad0  |        numbers; owned by CALLER
3550 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3551 //  h     ^        |   in   |  5
3552 //        |        |  args  |  4   Holes in incoming args owned by SELF
3553 //  |     |        |        |  3
3554 //  |     |        +--------+
3555 //  V     |        | old out|      Empty on Intel, window on Sparc
3556 //        |    old |preserve|      Must be even aligned.
3557 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3558 //        |        |   in   |  3   area for Intel ret address
3559 //     Owned by    |preserve|      Empty on Sparc.
3560 //       SELF      +--------+
3561 //        |        |  pad2  |  2   pad to align old SP
3562 //        |        +--------+  1
3563 //        |        | locks  |  0
3564 //        |        +--------+----> OptoReg::stack0(), even aligned
3565 //        |        |  pad1  | 11   pad to align new SP
3566 //        |        +--------+
3567 //        |        |        | 10
3568 //        |        | spills |  9   spills
3569 //        V        |        |  8   (pad0 slot for callee)
3570 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3571 //        ^        |  out   |  7
3572 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3573 //     Owned by    +--------+
3574 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3575 //        |    new |preserve|      Must be even-aligned.
3576 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3577 //        |        |        |
3578 //
3579 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3580 //         known from SELF's arguments and the Java calling convention.
3581 //         Region 6-7 is determined per call site.
3582 // Note 2: If the calling convention leaves holes in the incoming argument
3583 //         area, those holes are owned by SELF.  Holes in the outgoing area
3584 //         are owned by the CALLEE.  Holes should not be nessecary in the
3585 //         incoming area, as the Java calling convention is completely under
3586 //         the control of the AD file.  Doubles can be sorted and packed to
3587 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3588 //         varargs C calling conventions.
3589 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3590 //         even aligned with pad0 as needed.
3591 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3592 //           (the latter is true on Intel but is it false on AArch64?)
3593 //         region 6-11 is even aligned; it may be padded out more so that
3594 //         the region from SP to FP meets the minimum stack alignment.
3595 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3596 //         alignment.  Region 11, pad1, may be dynamically extended so that
3597 //         SP meets the minimum alignment.
3598 
3599 frame %{
3600   // What direction does stack grow in (assumed to be same for C & Java)
3601   stack_direction(TOWARDS_LOW);
3602 
3603   // These three registers define part of the calling convention
3604   // between compiled code and the interpreter.
3605 
3606   // Inline Cache Register or methodOop for I2C.
3607   inline_cache_reg(R12);
3608 
3609   // Method Oop Register when calling interpreter.
3610   interpreter_method_oop_reg(R12);
3611 
3612   // Number of stack slots consumed by locking an object
3613   sync_stack_slots(2);
3614 
3615   // Compiled code's Frame Pointer
3616   frame_pointer(R31);
3617 
3618   // Interpreter stores its frame pointer in a register which is
3619   // stored to the stack by I2CAdaptors.
3620   // I2CAdaptors convert from interpreted java to compiled java.
3621   interpreter_frame_pointer(R29);
3622 
3623   // Stack alignment requirement
3624   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3625 
3626   // Number of stack slots between incoming argument block and the start of
3627   // a new frame.  The PROLOG must add this many slots to the stack.  The
3628   // EPILOG must remove this many slots. aarch64 needs two slots for
3629   // return address and fp.
3630   // TODO think this is correct but check
3631   in_preserve_stack_slots(4);
3632 
3633   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3634   // for calls to C.  Supports the var-args backing area for register parms.
3635   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3636 
3637   // The after-PROLOG location of the return address.  Location of
3638   // return address specifies a type (REG or STACK) and a number
3639   // representing the register number (i.e. - use a register name) or
3640   // stack slot.
3641   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3642   // Otherwise, it is above the locks and verification slot and alignment word
3643   // TODO this may well be correct but need to check why that - 2 is there
3644   // ppc port uses 0 but we definitely need to allow for fixed_slots
3645   // which folds in the space used for monitors
3646   return_addr(STACK - 2 +
3647               align_up((Compile::current()->in_preserve_stack_slots() +
3648                         Compile::current()->fixed_slots()),
3649                        stack_alignment_in_slots()));
3650 
3651   // Body of function which returns an integer array locating
3652   // arguments either in registers or in stack slots.  Passed an array
3653   // of ideal registers called "sig" and a "length" count.  Stack-slot
3654   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3655   // arguments for a CALLEE.  Incoming stack arguments are
3656   // automatically biased by the preserve_stack_slots field above.
3657 
3658   calling_convention
3659   %{
3660     // No difference between ingoing/outgoing just pass false
3661     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3662   %}
3663 
3664   c_calling_convention
3665   %{
3666     // This is obviously always outgoing
3667     (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3668   %}
3669 
3670   // Location of compiled Java return values.  Same as C for now.
3671   return_value
3672   %{
3673     // TODO do we allow ideal_reg == Op_RegN???
3674     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3675            "only return normal values");
3676 
3677     static const int lo[Op_RegL + 1] = { // enum name
3678       0,                                 // Op_Node
3679       0,                                 // Op_Set
3680       R0_num,                            // Op_RegN
3681       R0_num,                            // Op_RegI
3682       R0_num,                            // Op_RegP
3683       V0_num,                            // Op_RegF
3684       V0_num,                            // Op_RegD
3685       R0_num                             // Op_RegL
3686     };
3687 
3688     static const int hi[Op_RegL + 1] = { // enum name
3689       0,                                 // Op_Node
3690       0,                                 // Op_Set
3691       OptoReg::Bad,                       // Op_RegN
3692       OptoReg::Bad,                      // Op_RegI
3693       R0_H_num,                          // Op_RegP
3694       OptoReg::Bad,                      // Op_RegF
3695       V0_H_num,                          // Op_RegD
3696       R0_H_num                           // Op_RegL
3697     };
3698 
3699     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3700   %}
3701 %}
3702 
3703 //----------ATTRIBUTES---------------------------------------------------------
3704 //----------Operand Attributes-------------------------------------------------
3705 op_attrib op_cost(1);        // Required cost attribute
3706 
3707 //----------Instruction Attributes---------------------------------------------
3708 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3709 ins_attrib ins_size(32);        // Required size attribute (in bits)
3710 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3711                                 // a non-matching short branch variant
3712                                 // of some long branch?
3713 ins_attrib ins_alignment(4);    // Required alignment attribute (must
3714                                 // be a power of 2) specifies the
3715                                 // alignment that some part of the
3716                                 // instruction (not necessarily the
3717                                 // start) requires.  If > 1, a
3718                                 // compute_padding() function must be
3719                                 // provided for the instruction
3720 
3721 //----------OPERANDS-----------------------------------------------------------
3722 // Operand definitions must precede instruction definitions for correct parsing
3723 // in the ADLC because operands constitute user defined types which are used in
3724 // instruction definitions.
3725 
3726 //----------Simple Operands----------------------------------------------------
3727 
3728 // Integer operands 32 bit
3729 // 32 bit immediate
3730 operand immI()
3731 %{
3732   match(ConI);
3733 
3734   op_cost(0);
3735   format %{ %}
3736   interface(CONST_INTER);
3737 %}
3738 
3739 // 32 bit zero
3740 operand immI0()
3741 %{
3742   predicate(n->get_int() == 0);
3743   match(ConI);
3744 
3745   op_cost(0);
3746   format %{ %}
3747   interface(CONST_INTER);
3748 %}
3749 
3750 // 32 bit unit increment
3751 operand immI_1()
3752 %{
3753   predicate(n->get_int() == 1);
3754   match(ConI);
3755 
3756   op_cost(0);
3757   format %{ %}
3758   interface(CONST_INTER);
3759 %}
3760 
3761 // 32 bit unit decrement
3762 operand immI_M1()
3763 %{
3764   predicate(n->get_int() == -1);
3765   match(ConI);
3766 
3767   op_cost(0);
3768   format %{ %}
3769   interface(CONST_INTER);
3770 %}
3771 
3772 // Shift values for add/sub extension shift
3773 operand immIExt()
3774 %{
3775   predicate(0 <= n->get_int() && (n->get_int() <= 4));
3776   match(ConI);
3777 
3778   op_cost(0);
3779   format %{ %}
3780   interface(CONST_INTER);
3781 %}
3782 
3783 operand immI_le_4()
3784 %{
3785   predicate(n->get_int() <= 4);
3786   match(ConI);
3787 
3788   op_cost(0);
3789   format %{ %}
3790   interface(CONST_INTER);
3791 %}
3792 
3793 operand immI_31()
3794 %{
3795   predicate(n->get_int() == 31);
3796   match(ConI);
3797 
3798   op_cost(0);
3799   format %{ %}
3800   interface(CONST_INTER);
3801 %}
3802 
3803 operand immI_8()
3804 %{
3805   predicate(n->get_int() == 8);
3806   match(ConI);
3807 
3808   op_cost(0);
3809   format %{ %}
3810   interface(CONST_INTER);
3811 %}
3812 
3813 operand immI_16()
3814 %{
3815   predicate(n->get_int() == 16);
3816   match(ConI);
3817 
3818   op_cost(0);
3819   format %{ %}
3820   interface(CONST_INTER);
3821 %}
3822 
3823 operand immI_24()
3824 %{
3825   predicate(n->get_int() == 24);
3826   match(ConI);
3827 
3828   op_cost(0);
3829   format %{ %}
3830   interface(CONST_INTER);
3831 %}
3832 
3833 operand immI_32()
3834 %{
3835   predicate(n->get_int() == 32);
3836   match(ConI);
3837 
3838   op_cost(0);
3839   format %{ %}
3840   interface(CONST_INTER);
3841 %}
3842 
3843 operand immI_48()
3844 %{
3845   predicate(n->get_int() == 48);
3846   match(ConI);
3847 
3848   op_cost(0);
3849   format %{ %}
3850   interface(CONST_INTER);
3851 %}
3852 
3853 operand immI_56()
3854 %{
3855   predicate(n->get_int() == 56);
3856   match(ConI);
3857 
3858   op_cost(0);
3859   format %{ %}
3860   interface(CONST_INTER);
3861 %}
3862 
3863 operand immI_63()
3864 %{
3865   predicate(n->get_int() == 63);
3866   match(ConI);
3867 
3868   op_cost(0);
3869   format %{ %}
3870   interface(CONST_INTER);
3871 %}
3872 
3873 operand immI_64()
3874 %{
3875   predicate(n->get_int() == 64);
3876   match(ConI);
3877 
3878   op_cost(0);
3879   format %{ %}
3880   interface(CONST_INTER);
3881 %}
3882 
3883 operand immI_255()
3884 %{
3885   predicate(n->get_int() == 255);
3886   match(ConI);
3887 
3888   op_cost(0);
3889   format %{ %}
3890   interface(CONST_INTER);
3891 %}
3892 
3893 operand immI_65535()
3894 %{
3895   predicate(n->get_int() == 65535);
3896   match(ConI);
3897 
3898   op_cost(0);
3899   format %{ %}
3900   interface(CONST_INTER);
3901 %}
3902 
3903 operand immL_255()
3904 %{
3905   predicate(n->get_long() == 255L);
3906   match(ConL);
3907 
3908   op_cost(0);
3909   format %{ %}
3910   interface(CONST_INTER);
3911 %}
3912 
3913 operand immL_65535()
3914 %{
3915   predicate(n->get_long() == 65535L);
3916   match(ConL);
3917 
3918   op_cost(0);
3919   format %{ %}
3920   interface(CONST_INTER);
3921 %}
3922 
3923 operand immL_4294967295()
3924 %{
3925   predicate(n->get_long() == 4294967295L);
3926   match(ConL);
3927 
3928   op_cost(0);
3929   format %{ %}
3930   interface(CONST_INTER);
3931 %}
3932 
3933 operand immL_bitmask()
3934 %{
3935   predicate((n->get_long() != 0)
3936             && ((n->get_long() & 0xc000000000000000l) == 0)
3937             && is_power_of_2(n->get_long() + 1));
3938   match(ConL);
3939 
3940   op_cost(0);
3941   format %{ %}
3942   interface(CONST_INTER);
3943 %}
3944 
3945 operand immI_bitmask()
3946 %{
3947   predicate((n->get_int() != 0)
3948             && ((n->get_int() & 0xc0000000) == 0)
3949             && is_power_of_2(n->get_int() + 1));
3950   match(ConI);
3951 
3952   op_cost(0);
3953   format %{ %}
3954   interface(CONST_INTER);
3955 %}
3956 
3957 // Scale values for scaled offset addressing modes (up to long but not quad)
3958 operand immIScale()
3959 %{
3960   predicate(0 <= n->get_int() && (n->get_int() <= 3));
3961   match(ConI);
3962 
3963   op_cost(0);
3964   format %{ %}
3965   interface(CONST_INTER);
3966 %}
3967 
3968 // 26 bit signed offset -- for pc-relative branches
3969 operand immI26()
3970 %{
3971   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
3972   match(ConI);
3973 
3974   op_cost(0);
3975   format %{ %}
3976   interface(CONST_INTER);
3977 %}
3978 
3979 // 19 bit signed offset -- for pc-relative loads
3980 operand immI19()
3981 %{
3982   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
3983   match(ConI);
3984 
3985   op_cost(0);
3986   format %{ %}
3987   interface(CONST_INTER);
3988 %}
3989 
3990 // 12 bit unsigned offset -- for base plus immediate loads
3991 operand immIU12()
3992 %{
3993   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
3994   match(ConI);
3995 
3996   op_cost(0);
3997   format %{ %}
3998   interface(CONST_INTER);
3999 %}
4000 
4001 operand immLU12()
4002 %{
4003   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4004   match(ConL);
4005 
4006   op_cost(0);
4007   format %{ %}
4008   interface(CONST_INTER);
4009 %}
4010 
4011 // Offset for scaled or unscaled immediate loads and stores
4012 operand immIOffset()
4013 %{
4014   predicate(Address::offset_ok_for_immed(n->get_int()));
4015   match(ConI);
4016 
4017   op_cost(0);
4018   format %{ %}
4019   interface(CONST_INTER);
4020 %}
4021 
4022 operand immIOffset4()
4023 %{
4024   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4025   match(ConI);
4026 
4027   op_cost(0);
4028   format %{ %}
4029   interface(CONST_INTER);
4030 %}
4031 
4032 operand immIOffset8()
4033 %{
4034   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4035   match(ConI);
4036 
4037   op_cost(0);
4038   format %{ %}
4039   interface(CONST_INTER);
4040 %}
4041 
4042 operand immIOffset16()
4043 %{
4044   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4045   match(ConI);
4046 
4047   op_cost(0);
4048   format %{ %}
4049   interface(CONST_INTER);
4050 %}
4051 
4052 operand immLoffset()
4053 %{
4054   predicate(Address::offset_ok_for_immed(n->get_long()));
4055   match(ConL);
4056 
4057   op_cost(0);
4058   format %{ %}
4059   interface(CONST_INTER);
4060 %}
4061 
4062 operand immLoffset4()
4063 %{
4064   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4065   match(ConL);
4066 
4067   op_cost(0);
4068   format %{ %}
4069   interface(CONST_INTER);
4070 %}
4071 
4072 operand immLoffset8()
4073 %{
4074   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4075   match(ConL);
4076 
4077   op_cost(0);
4078   format %{ %}
4079   interface(CONST_INTER);
4080 %}
4081 
4082 operand immLoffset16()
4083 %{
4084   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4085   match(ConL);
4086 
4087   op_cost(0);
4088   format %{ %}
4089   interface(CONST_INTER);
4090 %}
4091 
4092 // 32 bit integer valid for add sub immediate
4093 operand immIAddSub()
4094 %{
4095   predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
4096   match(ConI);
4097   op_cost(0);
4098   format %{ %}
4099   interface(CONST_INTER);
4100 %}
4101 
4102 // 32 bit unsigned integer valid for logical immediate
4103 // TODO -- check this is right when e.g the mask is 0x80000000
4104 operand immILog()
4105 %{
4106   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
4107   match(ConI);
4108 
4109   op_cost(0);
4110   format %{ %}
4111   interface(CONST_INTER);
4112 %}
4113 
4114 // Integer operands 64 bit
4115 // 64 bit immediate
4116 operand immL()
4117 %{
4118   match(ConL);
4119 
4120   op_cost(0);
4121   format %{ %}
4122   interface(CONST_INTER);
4123 %}
4124 
4125 // 64 bit zero
4126 operand immL0()
4127 %{
4128   predicate(n->get_long() == 0);
4129   match(ConL);
4130 
4131   op_cost(0);
4132   format %{ %}
4133   interface(CONST_INTER);
4134 %}
4135 
4136 // 64 bit unit increment
4137 operand immL_1()
4138 %{
4139   predicate(n->get_long() == 1);
4140   match(ConL);
4141 
4142   op_cost(0);
4143   format %{ %}
4144   interface(CONST_INTER);
4145 %}
4146 
4147 // 64 bit unit decrement
4148 operand immL_M1()
4149 %{
4150   predicate(n->get_long() == -1);
4151   match(ConL);
4152 
4153   op_cost(0);
4154   format %{ %}
4155   interface(CONST_INTER);
4156 %}
4157 
4158 // 32 bit offset of pc in thread anchor
4159 
4160 operand immL_pc_off()
4161 %{
4162   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4163                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4164   match(ConL);
4165 
4166   op_cost(0);
4167   format %{ %}
4168   interface(CONST_INTER);
4169 %}
4170 
4171 // 64 bit integer valid for add sub immediate
4172 operand immLAddSub()
4173 %{
4174   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4175   match(ConL);
4176   op_cost(0);
4177   format %{ %}
4178   interface(CONST_INTER);
4179 %}
4180 
4181 // 64 bit integer valid for logical immediate
4182 operand immLLog()
4183 %{
4184   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
4185   match(ConL);
4186   op_cost(0);
4187   format %{ %}
4188   interface(CONST_INTER);
4189 %}
4190 
4191 // Long Immediate: low 32-bit mask
4192 operand immL_32bits()
4193 %{
4194   predicate(n->get_long() == 0xFFFFFFFFL);
4195   match(ConL);
4196   op_cost(0);
4197   format %{ %}
4198   interface(CONST_INTER);
4199 %}
4200 
4201 // Pointer operands
4202 // Pointer Immediate
4203 operand immP()
4204 %{
4205   match(ConP);
4206 
4207   op_cost(0);
4208   format %{ %}
4209   interface(CONST_INTER);
4210 %}
4211 
4212 // NULL Pointer Immediate
4213 operand immP0()
4214 %{
4215   predicate(n->get_ptr() == 0);
4216   match(ConP);
4217 
4218   op_cost(0);
4219   format %{ %}
4220   interface(CONST_INTER);
4221 %}
4222 
4223 // Pointer Immediate One
4224 // this is used in object initialization (initial object header)
4225 operand immP_1()
4226 %{
4227   predicate(n->get_ptr() == 1);
4228   match(ConP);
4229 
4230   op_cost(0);
4231   format %{ %}
4232   interface(CONST_INTER);
4233 %}
4234 
4235 // Polling Page Pointer Immediate
4236 operand immPollPage()
4237 %{
4238   predicate((address)n->get_ptr() == os::get_polling_page());
4239   match(ConP);
4240 
4241   op_cost(0);
4242   format %{ %}
4243   interface(CONST_INTER);
4244 %}
4245 
4246 // Card Table Byte Map Base
4247 operand immByteMapBase()
4248 %{
4249   // Get base of card map
4250   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4251             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4252   match(ConP);
4253 
4254   op_cost(0);
4255   format %{ %}
4256   interface(CONST_INTER);
4257 %}
4258 
4259 // Pointer Immediate Minus One
4260 // this is used when we want to write the current PC to the thread anchor
4261 operand immP_M1()
4262 %{
4263   predicate(n->get_ptr() == -1);
4264   match(ConP);
4265 
4266   op_cost(0);
4267   format %{ %}
4268   interface(CONST_INTER);
4269 %}
4270 
4271 // Pointer Immediate Minus Two
4272 // this is used when we want to write the current PC to the thread anchor
4273 operand immP_M2()
4274 %{
4275   predicate(n->get_ptr() == -2);
4276   match(ConP);
4277 
4278   op_cost(0);
4279   format %{ %}
4280   interface(CONST_INTER);
4281 %}
4282 
4283 // Float and Double operands
4284 // Double Immediate
4285 operand immD()
4286 %{
4287   match(ConD);
4288   op_cost(0);
4289   format %{ %}
4290   interface(CONST_INTER);
4291 %}
4292 
4293 // Double Immediate: +0.0d
4294 operand immD0()
4295 %{
4296   predicate(jlong_cast(n->getd()) == 0);
4297   match(ConD);
4298 
4299   op_cost(0);
4300   format %{ %}
4301   interface(CONST_INTER);
4302 %}
4303 
4304 // constant 'double +0.0'.
4305 operand immDPacked()
4306 %{
4307   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4308   match(ConD);
4309   op_cost(0);
4310   format %{ %}
4311   interface(CONST_INTER);
4312 %}
4313 
4314 // Float Immediate
4315 operand immF()
4316 %{
4317   match(ConF);
4318   op_cost(0);
4319   format %{ %}
4320   interface(CONST_INTER);
4321 %}
4322 
4323 // Float Immediate: +0.0f.
4324 operand immF0()
4325 %{
4326   predicate(jint_cast(n->getf()) == 0);
4327   match(ConF);
4328 
4329   op_cost(0);
4330   format %{ %}
4331   interface(CONST_INTER);
4332 %}
4333 
4334 //
4335 operand immFPacked()
4336 %{
4337   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4338   match(ConF);
4339   op_cost(0);
4340   format %{ %}
4341   interface(CONST_INTER);
4342 %}
4343 
4344 // Narrow pointer operands
4345 // Narrow Pointer Immediate
4346 operand immN()
4347 %{
4348   match(ConN);
4349 
4350   op_cost(0);
4351   format %{ %}
4352   interface(CONST_INTER);
4353 %}
4354 
4355 // Narrow NULL Pointer Immediate
4356 operand immN0()
4357 %{
4358   predicate(n->get_narrowcon() == 0);
4359   match(ConN);
4360 
4361   op_cost(0);
4362   format %{ %}
4363   interface(CONST_INTER);
4364 %}
4365 
4366 operand immNKlass()
4367 %{
4368   match(ConNKlass);
4369 
4370   op_cost(0);
4371   format %{ %}
4372   interface(CONST_INTER);
4373 %}
4374 
4375 // Integer 32 bit Register Operands
4376 // Integer 32 bitRegister (excludes SP)
4377 operand iRegI()
4378 %{
4379   constraint(ALLOC_IN_RC(any_reg32));
4380   match(RegI);
4381   match(iRegINoSp);
4382   op_cost(0);
4383   format %{ %}
4384   interface(REG_INTER);
4385 %}
4386 
4387 // Integer 32 bit Register not Special
4388 operand iRegINoSp()
4389 %{
4390   constraint(ALLOC_IN_RC(no_special_reg32));
4391   match(RegI);
4392   op_cost(0);
4393   format %{ %}
4394   interface(REG_INTER);
4395 %}
4396 
4397 // Integer 64 bit Register Operands
4398 // Integer 64 bit Register (includes SP)
4399 operand iRegL()
4400 %{
4401   constraint(ALLOC_IN_RC(any_reg));
4402   match(RegL);
4403   match(iRegLNoSp);
4404   op_cost(0);
4405   format %{ %}
4406   interface(REG_INTER);
4407 %}
4408 
4409 // Integer 64 bit Register not Special
4410 operand iRegLNoSp()
4411 %{
4412   constraint(ALLOC_IN_RC(no_special_reg));
4413   match(RegL);
4414   match(iRegL_R0);
4415   format %{ %}
4416   interface(REG_INTER);
4417 %}
4418 
4419 // Pointer Register Operands
4420 // Pointer Register
4421 operand iRegP()
4422 %{
4423   constraint(ALLOC_IN_RC(ptr_reg));
4424   match(RegP);
4425   match(iRegPNoSp);
4426   match(iRegP_R0);
4427   //match(iRegP_R2);
4428   //match(iRegP_R4);
4429   //match(iRegP_R5);
4430   match(thread_RegP);
4431   op_cost(0);
4432   format %{ %}
4433   interface(REG_INTER);
4434 %}
4435 
4436 // Pointer 64 bit Register not Special
4437 operand iRegPNoSp()
4438 %{
4439   constraint(ALLOC_IN_RC(no_special_ptr_reg));
4440   match(RegP);
4441   // match(iRegP);
4442   // match(iRegP_R0);
4443   // match(iRegP_R2);
4444   // match(iRegP_R4);
4445   // match(iRegP_R5);
4446   // match(thread_RegP);
4447   op_cost(0);
4448   format %{ %}
4449   interface(REG_INTER);
4450 %}
4451 
4452 // Pointer 64 bit Register R0 only
4453 operand iRegP_R0()
4454 %{
4455   constraint(ALLOC_IN_RC(r0_reg));
4456   match(RegP);
4457   // match(iRegP);
4458   match(iRegPNoSp);
4459   op_cost(0);
4460   format %{ %}
4461   interface(REG_INTER);
4462 %}
4463 
4464 // Pointer 64 bit Register R1 only
4465 operand iRegP_R1()
4466 %{
4467   constraint(ALLOC_IN_RC(r1_reg));
4468   match(RegP);
4469   // match(iRegP);
4470   match(iRegPNoSp);
4471   op_cost(0);
4472   format %{ %}
4473   interface(REG_INTER);
4474 %}
4475 
4476 // Pointer 64 bit Register R2 only
4477 operand iRegP_R2()
4478 %{
4479   constraint(ALLOC_IN_RC(r2_reg));
4480   match(RegP);
4481   // match(iRegP);
4482   match(iRegPNoSp);
4483   op_cost(0);
4484   format %{ %}
4485   interface(REG_INTER);
4486 %}
4487 
4488 // Pointer 64 bit Register R3 only
4489 operand iRegP_R3()
4490 %{
4491   constraint(ALLOC_IN_RC(r3_reg));
4492   match(RegP);
4493   // match(iRegP);
4494   match(iRegPNoSp);
4495   op_cost(0);
4496   format %{ %}
4497   interface(REG_INTER);
4498 %}
4499 
4500 // Pointer 64 bit Register R4 only
4501 operand iRegP_R4()
4502 %{
4503   constraint(ALLOC_IN_RC(r4_reg));
4504   match(RegP);
4505   // match(iRegP);
4506   match(iRegPNoSp);
4507   op_cost(0);
4508   format %{ %}
4509   interface(REG_INTER);
4510 %}
4511 
4512 // Pointer 64 bit Register R5 only
4513 operand iRegP_R5()
4514 %{
4515   constraint(ALLOC_IN_RC(r5_reg));
4516   match(RegP);
4517   // match(iRegP);
4518   match(iRegPNoSp);
4519   op_cost(0);
4520   format %{ %}
4521   interface(REG_INTER);
4522 %}
4523 
4524 // Pointer 64 bit Register R10 only
4525 operand iRegP_R10()
4526 %{
4527   constraint(ALLOC_IN_RC(r10_reg));
4528   match(RegP);
4529   // match(iRegP);
4530   match(iRegPNoSp);
4531   op_cost(0);
4532   format %{ %}
4533   interface(REG_INTER);
4534 %}
4535 
4536 // Long 64 bit Register R0 only
4537 operand iRegL_R0()
4538 %{
4539   constraint(ALLOC_IN_RC(r0_reg));
4540   match(RegL);
4541   match(iRegLNoSp);
4542   op_cost(0);
4543   format %{ %}
4544   interface(REG_INTER);
4545 %}
4546 
4547 // Long 64 bit Register R2 only
4548 operand iRegL_R2()
4549 %{
4550   constraint(ALLOC_IN_RC(r2_reg));
4551   match(RegL);
4552   match(iRegLNoSp);
4553   op_cost(0);
4554   format %{ %}
4555   interface(REG_INTER);
4556 %}
4557 
4558 // Long 64 bit Register R3 only
4559 operand iRegL_R3()
4560 %{
4561   constraint(ALLOC_IN_RC(r3_reg));
4562   match(RegL);
4563   match(iRegLNoSp);
4564   op_cost(0);
4565   format %{ %}
4566   interface(REG_INTER);
4567 %}
4568 
4569 // Long 64 bit Register R11 only
4570 operand iRegL_R11()
4571 %{
4572   constraint(ALLOC_IN_RC(r11_reg));
4573   match(RegL);
4574   match(iRegLNoSp);
4575   op_cost(0);
4576   format %{ %}
4577   interface(REG_INTER);
4578 %}
4579 
4580 // Pointer 64 bit Register FP only
4581 operand iRegP_FP()
4582 %{
4583   constraint(ALLOC_IN_RC(fp_reg));
4584   match(RegP);
4585   // match(iRegP);
4586   op_cost(0);
4587   format %{ %}
4588   interface(REG_INTER);
4589 %}
4590 
4591 // Register R0 only
4592 operand iRegI_R0()
4593 %{
4594   constraint(ALLOC_IN_RC(int_r0_reg));
4595   match(RegI);
4596   match(iRegINoSp);
4597   op_cost(0);
4598   format %{ %}
4599   interface(REG_INTER);
4600 %}
4601 
4602 // Register R2 only
4603 operand iRegI_R2()
4604 %{
4605   constraint(ALLOC_IN_RC(int_r2_reg));
4606   match(RegI);
4607   match(iRegINoSp);
4608   op_cost(0);
4609   format %{ %}
4610   interface(REG_INTER);
4611 %}
4612 
4613 // Register R3 only
4614 operand iRegI_R3()
4615 %{
4616   constraint(ALLOC_IN_RC(int_r3_reg));
4617   match(RegI);
4618   match(iRegINoSp);
4619   op_cost(0);
4620   format %{ %}
4621   interface(REG_INTER);
4622 %}
4623 
4624 
4625 // Register R4 only
4626 operand iRegI_R4()
4627 %{
4628   constraint(ALLOC_IN_RC(int_r4_reg));
4629   match(RegI);
4630   match(iRegINoSp);
4631   op_cost(0);
4632   format %{ %}
4633   interface(REG_INTER);
4634 %}
4635 
4636 
4637 // Pointer Register Operands
4638 // Narrow Pointer Register
4639 operand iRegN()
4640 %{
4641   constraint(ALLOC_IN_RC(any_reg32));
4642   match(RegN);
4643   match(iRegNNoSp);
4644   op_cost(0);
4645   format %{ %}
4646   interface(REG_INTER);
4647 %}
4648 
4649 operand iRegN_R0()
4650 %{
4651   constraint(ALLOC_IN_RC(r0_reg));
4652   match(iRegN);
4653   op_cost(0);
4654   format %{ %}
4655   interface(REG_INTER);
4656 %}
4657 
4658 operand iRegN_R2()
4659 %{
4660   constraint(ALLOC_IN_RC(r2_reg));
4661   match(iRegN);
4662   op_cost(0);
4663   format %{ %}
4664   interface(REG_INTER);
4665 %}
4666 
4667 operand iRegN_R3()
4668 %{
4669   constraint(ALLOC_IN_RC(r3_reg));
4670   match(iRegN);
4671   op_cost(0);
4672   format %{ %}
4673   interface(REG_INTER);
4674 %}
4675 
4676 // Integer 64 bit Register not Special
4677 operand iRegNNoSp()
4678 %{
4679   constraint(ALLOC_IN_RC(no_special_reg32));
4680   match(RegN);
4681   op_cost(0);
4682   format %{ %}
4683   interface(REG_INTER);
4684 %}
4685 
4686 // heap base register -- used for encoding immN0
4687 
4688 operand iRegIHeapbase()
4689 %{
4690   constraint(ALLOC_IN_RC(heapbase_reg));
4691   match(RegI);
4692   op_cost(0);
4693   format %{ %}
4694   interface(REG_INTER);
4695 %}
4696 
4697 // Float Register
4698 // Float register operands
4699 operand vRegF()
4700 %{
4701   constraint(ALLOC_IN_RC(float_reg));
4702   match(RegF);
4703 
4704   op_cost(0);
4705   format %{ %}
4706   interface(REG_INTER);
4707 %}
4708 
4709 // Double Register
4710 // Double register operands
4711 operand vRegD()
4712 %{
4713   constraint(ALLOC_IN_RC(double_reg));
4714   match(RegD);
4715 
4716   op_cost(0);
4717   format %{ %}
4718   interface(REG_INTER);
4719 %}
4720 
4721 operand vecD()
4722 %{
4723   constraint(ALLOC_IN_RC(vectord_reg));
4724   match(VecD);
4725 
4726   op_cost(0);
4727   format %{ %}
4728   interface(REG_INTER);
4729 %}
4730 
4731 operand vecX()
4732 %{
4733   constraint(ALLOC_IN_RC(vectorx_reg));
4734   match(VecX);
4735 
4736   op_cost(0);
4737   format %{ %}
4738   interface(REG_INTER);
4739 %}
4740 
4741 operand vRegD_V0()
4742 %{
4743   constraint(ALLOC_IN_RC(v0_reg));
4744   match(RegD);
4745   op_cost(0);
4746   format %{ %}
4747   interface(REG_INTER);
4748 %}
4749 
4750 operand vRegD_V1()
4751 %{
4752   constraint(ALLOC_IN_RC(v1_reg));
4753   match(RegD);
4754   op_cost(0);
4755   format %{ %}
4756   interface(REG_INTER);
4757 %}
4758 
4759 operand vRegD_V2()
4760 %{
4761   constraint(ALLOC_IN_RC(v2_reg));
4762   match(RegD);
4763   op_cost(0);
4764   format %{ %}
4765   interface(REG_INTER);
4766 %}
4767 
4768 operand vRegD_V3()
4769 %{
4770   constraint(ALLOC_IN_RC(v3_reg));
4771   match(RegD);
4772   op_cost(0);
4773   format %{ %}
4774   interface(REG_INTER);
4775 %}
4776 
4777 // Flags register, used as output of signed compare instructions
4778 
4779 // note that on AArch64 we also use this register as the output for
4780 // for floating point compare instructions (CmpF CmpD). this ensures
4781 // that ordered inequality tests use GT, GE, LT or LE none of which
4782 // pass through cases where the result is unordered i.e. one or both
4783 // inputs to the compare is a NaN. this means that the ideal code can
4784 // replace e.g. a GT with an LE and not end up capturing the NaN case
4785 // (where the comparison should always fail). EQ and NE tests are
4786 // always generated in ideal code so that unordered folds into the NE
4787 // case, matching the behaviour of AArch64 NE.
4788 //
4789 // This differs from x86 where the outputs of FP compares use a
4790 // special FP flags registers and where compares based on this
4791 // register are distinguished into ordered inequalities (cmpOpUCF) and
4792 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
4793 // to explicitly handle the unordered case in branches. x86 also has
4794 // to include extra CMoveX rules to accept a cmpOpUCF input.
4795 
4796 operand rFlagsReg()
4797 %{
4798   constraint(ALLOC_IN_RC(int_flags));
4799   match(RegFlags);
4800 
4801   op_cost(0);
4802   format %{ "RFLAGS" %}
4803   interface(REG_INTER);
4804 %}
4805 
4806 // Flags register, used as output of unsigned compare instructions
4807 operand rFlagsRegU()
4808 %{
4809   constraint(ALLOC_IN_RC(int_flags));
4810   match(RegFlags);
4811 
4812   op_cost(0);
4813   format %{ "RFLAGSU" %}
4814   interface(REG_INTER);
4815 %}
4816 
4817 // Special Registers
4818 
4819 // Method Register
4820 operand inline_cache_RegP(iRegP reg)
4821 %{
4822   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
4823   match(reg);
4824   match(iRegPNoSp);
4825   op_cost(0);
4826   format %{ %}
4827   interface(REG_INTER);
4828 %}
4829 
4830 operand interpreter_method_oop_RegP(iRegP reg)
4831 %{
4832   constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
4833   match(reg);
4834   match(iRegPNoSp);
4835   op_cost(0);
4836   format %{ %}
4837   interface(REG_INTER);
4838 %}
4839 
4840 // Thread Register
4841 operand thread_RegP(iRegP reg)
4842 %{
4843   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
4844   match(reg);
4845   op_cost(0);
4846   format %{ %}
4847   interface(REG_INTER);
4848 %}
4849 
4850 operand lr_RegP(iRegP reg)
4851 %{
4852   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
4853   match(reg);
4854   op_cost(0);
4855   format %{ %}
4856   interface(REG_INTER);
4857 %}
4858 
4859 //----------Memory Operands----------------------------------------------------
4860 
4861 operand indirect(iRegP reg)
4862 %{
4863   constraint(ALLOC_IN_RC(ptr_reg));
4864   match(reg);
4865   op_cost(0);
4866   format %{ "[$reg]" %}
4867   interface(MEMORY_INTER) %{
4868     base($reg);
4869     index(0xffffffff);
4870     scale(0x0);
4871     disp(0x0);
4872   %}
4873 %}
4874 
4875 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
4876 %{
4877   constraint(ALLOC_IN_RC(ptr_reg));
4878   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
4879   match(AddP reg (LShiftL (ConvI2L ireg) scale));
4880   op_cost(0);
4881   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
4882   interface(MEMORY_INTER) %{
4883     base($reg);
4884     index($ireg);
4885     scale($scale);
4886     disp(0x0);
4887   %}
4888 %}
4889 
4890 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
4891 %{
4892   constraint(ALLOC_IN_RC(ptr_reg));
4893   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
4894   match(AddP reg (LShiftL lreg scale));
4895   op_cost(0);
4896   format %{ "$reg, $lreg lsl($scale)" %}
4897   interface(MEMORY_INTER) %{
4898     base($reg);
4899     index($lreg);
4900     scale($scale);
4901     disp(0x0);
4902   %}
4903 %}
4904 
4905 operand indIndexI2L(iRegP reg, iRegI ireg)
4906 %{
4907   constraint(ALLOC_IN_RC(ptr_reg));
4908   match(AddP reg (ConvI2L ireg));
4909   op_cost(0);
4910   format %{ "$reg, $ireg, 0, I2L" %}
4911   interface(MEMORY_INTER) %{
4912     base($reg);
4913     index($ireg);
4914     scale(0x0);
4915     disp(0x0);
4916   %}
4917 %}
4918 
4919 operand indIndex(iRegP reg, iRegL lreg)
4920 %{
4921   constraint(ALLOC_IN_RC(ptr_reg));
4922   match(AddP reg lreg);
4923   op_cost(0);
4924   format %{ "$reg, $lreg" %}
4925   interface(MEMORY_INTER) %{
4926     base($reg);
4927     index($lreg);
4928     scale(0x0);
4929     disp(0x0);
4930   %}
4931 %}
4932 
4933 operand indOffI(iRegP reg, immIOffset off)
4934 %{
4935   constraint(ALLOC_IN_RC(ptr_reg));
4936   match(AddP reg off);
4937   op_cost(0);
4938   format %{ "[$reg, $off]" %}
4939   interface(MEMORY_INTER) %{
4940     base($reg);
4941     index(0xffffffff);
4942     scale(0x0);
4943     disp($off);
4944   %}
4945 %}
4946 
4947 operand indOffI4(iRegP reg, immIOffset4 off)
4948 %{
4949   constraint(ALLOC_IN_RC(ptr_reg));
4950   match(AddP reg off);
4951   op_cost(0);
4952   format %{ "[$reg, $off]" %}
4953   interface(MEMORY_INTER) %{
4954     base($reg);
4955     index(0xffffffff);
4956     scale(0x0);
4957     disp($off);
4958   %}
4959 %}
4960 
4961 operand indOffI8(iRegP reg, immIOffset8 off)
4962 %{
4963   constraint(ALLOC_IN_RC(ptr_reg));
4964   match(AddP reg off);
4965   op_cost(0);
4966   format %{ "[$reg, $off]" %}
4967   interface(MEMORY_INTER) %{
4968     base($reg);
4969     index(0xffffffff);
4970     scale(0x0);
4971     disp($off);
4972   %}
4973 %}
4974 
4975 operand indOffI16(iRegP reg, immIOffset16 off)
4976 %{
4977   constraint(ALLOC_IN_RC(ptr_reg));
4978   match(AddP reg off);
4979   op_cost(0);
4980   format %{ "[$reg, $off]" %}
4981   interface(MEMORY_INTER) %{
4982     base($reg);
4983     index(0xffffffff);
4984     scale(0x0);
4985     disp($off);
4986   %}
4987 %}
4988 
4989 operand indOffL(iRegP reg, immLoffset off)
4990 %{
4991   constraint(ALLOC_IN_RC(ptr_reg));
4992   match(AddP reg off);
4993   op_cost(0);
4994   format %{ "[$reg, $off]" %}
4995   interface(MEMORY_INTER) %{
4996     base($reg);
4997     index(0xffffffff);
4998     scale(0x0);
4999     disp($off);
5000   %}
5001 %}
5002 
5003 operand indOffL4(iRegP reg, immLoffset4 off)
5004 %{
5005   constraint(ALLOC_IN_RC(ptr_reg));
5006   match(AddP reg off);
5007   op_cost(0);
5008   format %{ "[$reg, $off]" %}
5009   interface(MEMORY_INTER) %{
5010     base($reg);
5011     index(0xffffffff);
5012     scale(0x0);
5013     disp($off);
5014   %}
5015 %}
5016 
5017 operand indOffL8(iRegP reg, immLoffset8 off)
5018 %{
5019   constraint(ALLOC_IN_RC(ptr_reg));
5020   match(AddP reg off);
5021   op_cost(0);
5022   format %{ "[$reg, $off]" %}
5023   interface(MEMORY_INTER) %{
5024     base($reg);
5025     index(0xffffffff);
5026     scale(0x0);
5027     disp($off);
5028   %}
5029 %}
5030 
5031 operand indOffL16(iRegP reg, immLoffset16 off)
5032 %{
5033   constraint(ALLOC_IN_RC(ptr_reg));
5034   match(AddP reg off);
5035   op_cost(0);
5036   format %{ "[$reg, $off]" %}
5037   interface(MEMORY_INTER) %{
5038     base($reg);
5039     index(0xffffffff);
5040     scale(0x0);
5041     disp($off);
5042   %}
5043 %}
5044 
5045 operand indirectN(iRegN reg)
5046 %{
5047   predicate(CompressedOops::shift() == 0);
5048   constraint(ALLOC_IN_RC(ptr_reg));
5049   match(DecodeN reg);
5050   op_cost(0);
5051   format %{ "[$reg]\t# narrow" %}
5052   interface(MEMORY_INTER) %{
5053     base($reg);
5054     index(0xffffffff);
5055     scale(0x0);
5056     disp(0x0);
5057   %}
5058 %}
5059 
5060 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5061 %{
5062   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5063   constraint(ALLOC_IN_RC(ptr_reg));
5064   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5065   op_cost(0);
5066   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5067   interface(MEMORY_INTER) %{
5068     base($reg);
5069     index($ireg);
5070     scale($scale);
5071     disp(0x0);
5072   %}
5073 %}
5074 
5075 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5076 %{
5077   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5078   constraint(ALLOC_IN_RC(ptr_reg));
5079   match(AddP (DecodeN reg) (LShiftL lreg scale));
5080   op_cost(0);
5081   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5082   interface(MEMORY_INTER) %{
5083     base($reg);
5084     index($lreg);
5085     scale($scale);
5086     disp(0x0);
5087   %}
5088 %}
5089 
5090 operand indIndexI2LN(iRegN reg, iRegI ireg)
5091 %{
5092   predicate(CompressedOops::shift() == 0);
5093   constraint(ALLOC_IN_RC(ptr_reg));
5094   match(AddP (DecodeN reg) (ConvI2L ireg));
5095   op_cost(0);
5096   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5097   interface(MEMORY_INTER) %{
5098     base($reg);
5099     index($ireg);
5100     scale(0x0);
5101     disp(0x0);
5102   %}
5103 %}
5104 
5105 operand indIndexN(iRegN reg, iRegL lreg)
5106 %{
5107   predicate(CompressedOops::shift() == 0);
5108   constraint(ALLOC_IN_RC(ptr_reg));
5109   match(AddP (DecodeN reg) lreg);
5110   op_cost(0);
5111   format %{ "$reg, $lreg\t# narrow" %}
5112   interface(MEMORY_INTER) %{
5113     base($reg);
5114     index($lreg);
5115     scale(0x0);
5116     disp(0x0);
5117   %}
5118 %}
5119 
5120 operand indOffIN(iRegN reg, immIOffset off)
5121 %{
5122   predicate(CompressedOops::shift() == 0);
5123   constraint(ALLOC_IN_RC(ptr_reg));
5124   match(AddP (DecodeN reg) off);
5125   op_cost(0);
5126   format %{ "[$reg, $off]\t# narrow" %}
5127   interface(MEMORY_INTER) %{
5128     base($reg);
5129     index(0xffffffff);
5130     scale(0x0);
5131     disp($off);
5132   %}
5133 %}
5134 
5135 operand indOffLN(iRegN reg, immLoffset off)
5136 %{
5137   predicate(CompressedOops::shift() == 0);
5138   constraint(ALLOC_IN_RC(ptr_reg));
5139   match(AddP (DecodeN reg) off);
5140   op_cost(0);
5141   format %{ "[$reg, $off]\t# narrow" %}
5142   interface(MEMORY_INTER) %{
5143     base($reg);
5144     index(0xffffffff);
5145     scale(0x0);
5146     disp($off);
5147   %}
5148 %}
5149 
5150 
5151 
5152 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5153 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5154 %{
5155   constraint(ALLOC_IN_RC(ptr_reg));
5156   match(AddP reg off);
5157   op_cost(0);
5158   format %{ "[$reg, $off]" %}
5159   interface(MEMORY_INTER) %{
5160     base($reg);
5161     index(0xffffffff);
5162     scale(0x0);
5163     disp($off);
5164   %}
5165 %}
5166 
5167 //----------Special Memory Operands--------------------------------------------
5168 // Stack Slot Operand - This operand is used for loading and storing temporary
5169 //                      values on the stack where a match requires a value to
5170 //                      flow through memory.
5171 operand stackSlotP(sRegP reg)
5172 %{
5173   constraint(ALLOC_IN_RC(stack_slots));
5174   op_cost(100);
5175   // No match rule because this operand is only generated in matching
5176   // match(RegP);
5177   format %{ "[$reg]" %}
5178   interface(MEMORY_INTER) %{
5179     base(0x1e);  // RSP
5180     index(0x0);  // No Index
5181     scale(0x0);  // No Scale
5182     disp($reg);  // Stack Offset
5183   %}
5184 %}
5185 
5186 operand stackSlotI(sRegI reg)
5187 %{
5188   constraint(ALLOC_IN_RC(stack_slots));
5189   // No match rule because this operand is only generated in matching
5190   // match(RegI);
5191   format %{ "[$reg]" %}
5192   interface(MEMORY_INTER) %{
5193     base(0x1e);  // RSP
5194     index(0x0);  // No Index
5195     scale(0x0);  // No Scale
5196     disp($reg);  // Stack Offset
5197   %}
5198 %}
5199 
5200 operand stackSlotF(sRegF reg)
5201 %{
5202   constraint(ALLOC_IN_RC(stack_slots));
5203   // No match rule because this operand is only generated in matching
5204   // match(RegF);
5205   format %{ "[$reg]" %}
5206   interface(MEMORY_INTER) %{
5207     base(0x1e);  // RSP
5208     index(0x0);  // No Index
5209     scale(0x0);  // No Scale
5210     disp($reg);  // Stack Offset
5211   %}
5212 %}
5213 
5214 operand stackSlotD(sRegD reg)
5215 %{
5216   constraint(ALLOC_IN_RC(stack_slots));
5217   // No match rule because this operand is only generated in matching
5218   // match(RegD);
5219   format %{ "[$reg]" %}
5220   interface(MEMORY_INTER) %{
5221     base(0x1e);  // RSP
5222     index(0x0);  // No Index
5223     scale(0x0);  // No Scale
5224     disp($reg);  // Stack Offset
5225   %}
5226 %}
5227 
5228 operand stackSlotL(sRegL reg)
5229 %{
5230   constraint(ALLOC_IN_RC(stack_slots));
5231   // No match rule because this operand is only generated in matching
5232   // match(RegL);
5233   format %{ "[$reg]" %}
5234   interface(MEMORY_INTER) %{
5235     base(0x1e);  // RSP
5236     index(0x0);  // No Index
5237     scale(0x0);  // No Scale
5238     disp($reg);  // Stack Offset
5239   %}
5240 %}
5241 
5242 // Operands for expressing Control Flow
5243 // NOTE: Label is a predefined operand which should not be redefined in
5244 //       the AD file. It is generically handled within the ADLC.
5245 
5246 //----------Conditional Branch Operands----------------------------------------
5247 // Comparison Op  - This is the operation of the comparison, and is limited to
5248 //                  the following set of codes:
5249 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5250 //
5251 // Other attributes of the comparison, such as unsignedness, are specified
5252 // by the comparison instruction that sets a condition code flags register.
5253 // That result is represented by a flags operand whose subtype is appropriate
5254 // to the unsignedness (etc.) of the comparison.
5255 //
5256 // Later, the instruction which matches both the Comparison Op (a Bool) and
5257 // the flags (produced by the Cmp) specifies the coding of the comparison op
5258 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5259 
5260 // used for signed integral comparisons and fp comparisons
5261 
5262 operand cmpOp()
5263 %{
5264   match(Bool);
5265 
5266   format %{ "" %}
5267   interface(COND_INTER) %{
5268     equal(0x0, "eq");
5269     not_equal(0x1, "ne");
5270     less(0xb, "lt");
5271     greater_equal(0xa, "ge");
5272     less_equal(0xd, "le");
5273     greater(0xc, "gt");
5274     overflow(0x6, "vs");
5275     no_overflow(0x7, "vc");
5276   %}
5277 %}
5278 
5279 // used for unsigned integral comparisons
5280 
5281 operand cmpOpU()
5282 %{
5283   match(Bool);
5284 
5285   format %{ "" %}
5286   interface(COND_INTER) %{
5287     equal(0x0, "eq");
5288     not_equal(0x1, "ne");
5289     less(0x3, "lo");
5290     greater_equal(0x2, "hs");
5291     less_equal(0x9, "ls");
5292     greater(0x8, "hi");
5293     overflow(0x6, "vs");
5294     no_overflow(0x7, "vc");
5295   %}
5296 %}
5297 
5298 // used for certain integral comparisons which can be
5299 // converted to cbxx or tbxx instructions
5300 
5301 operand cmpOpEqNe()
5302 %{
5303   match(Bool);
5304   match(CmpOp);
5305   op_cost(0);
5306   predicate(n->as_Bool()->_test._test == BoolTest::ne
5307             || n->as_Bool()->_test._test == BoolTest::eq);
5308 
5309   format %{ "" %}
5310   interface(COND_INTER) %{
5311     equal(0x0, "eq");
5312     not_equal(0x1, "ne");
5313     less(0xb, "lt");
5314     greater_equal(0xa, "ge");
5315     less_equal(0xd, "le");
5316     greater(0xc, "gt");
5317     overflow(0x6, "vs");
5318     no_overflow(0x7, "vc");
5319   %}
5320 %}
5321 
5322 // used for certain integral comparisons which can be
5323 // converted to cbxx or tbxx instructions
5324 
5325 operand cmpOpLtGe()
5326 %{
5327   match(Bool);
5328   match(CmpOp);
5329   op_cost(0);
5330 
5331   predicate(n->as_Bool()->_test._test == BoolTest::lt
5332             || n->as_Bool()->_test._test == BoolTest::ge);
5333 
5334   format %{ "" %}
5335   interface(COND_INTER) %{
5336     equal(0x0, "eq");
5337     not_equal(0x1, "ne");
5338     less(0xb, "lt");
5339     greater_equal(0xa, "ge");
5340     less_equal(0xd, "le");
5341     greater(0xc, "gt");
5342     overflow(0x6, "vs");
5343     no_overflow(0x7, "vc");
5344   %}
5345 %}
5346 
5347 // used for certain unsigned integral comparisons which can be
5348 // converted to cbxx or tbxx instructions
5349 
5350 operand cmpOpUEqNeLtGe()
5351 %{
5352   match(Bool);
5353   match(CmpOp);
5354   op_cost(0);
5355 
5356   predicate(n->as_Bool()->_test._test == BoolTest::eq
5357             || n->as_Bool()->_test._test == BoolTest::ne
5358             || n->as_Bool()->_test._test == BoolTest::lt
5359             || n->as_Bool()->_test._test == BoolTest::ge);
5360 
5361   format %{ "" %}
5362   interface(COND_INTER) %{
5363     equal(0x0, "eq");
5364     not_equal(0x1, "ne");
5365     less(0xb, "lt");
5366     greater_equal(0xa, "ge");
5367     less_equal(0xd, "le");
5368     greater(0xc, "gt");
5369     overflow(0x6, "vs");
5370     no_overflow(0x7, "vc");
5371   %}
5372 %}
5373 
5374 // Special operand allowing long args to int ops to be truncated for free
5375 
5376 operand iRegL2I(iRegL reg) %{
5377 
5378   op_cost(0);
5379 
5380   match(ConvL2I reg);
5381 
5382   format %{ "l2i($reg)" %}
5383 
5384   interface(REG_INTER)
5385 %}
5386 
5387 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5388 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5389 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5390 
5391 //----------OPERAND CLASSES----------------------------------------------------
5392 // Operand Classes are groups of operands that are used as to simplify
5393 // instruction definitions by not requiring the AD writer to specify
5394 // separate instructions for every form of operand when the
5395 // instruction accepts multiple operand types with the same basic
5396 // encoding and format. The classic case of this is memory operands.
5397 
5398 // memory is used to define read/write location for load/store
5399 // instruction defs. we can turn a memory op into an Address
5400 
5401 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI, indOffL,
5402                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5403 
5404 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5405 // operations. it allows the src to be either an iRegI or a (ConvL2I
5406 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5407 // can be elided because the 32-bit instruction will just employ the
5408 // lower 32 bits anyway.
5409 //
5410 // n.b. this does not elide all L2I conversions. if the truncated
5411 // value is consumed by more than one operation then the ConvL2I
5412 // cannot be bundled into the consuming nodes so an l2i gets planted
5413 // (actually a movw $dst $src) and the downstream instructions consume
5414 // the result of the l2i as an iRegI input. That's a shame since the
5415 // movw is actually redundant but its not too costly.
5416 
5417 opclass iRegIorL2I(iRegI, iRegL2I);
5418 
5419 //----------PIPELINE-----------------------------------------------------------
5420 // Rules which define the behavior of the target architectures pipeline.
5421 
5422 // For specific pipelines, eg A53, define the stages of that pipeline
5423 //pipe_desc(ISS, EX1, EX2, WR);
5424 #define ISS S0
5425 #define EX1 S1
5426 #define EX2 S2
5427 #define WR  S3
5428 
5429 // Integer ALU reg operation
5430 pipeline %{
5431 
5432 attributes %{
5433   // ARM instructions are of fixed length
5434   fixed_size_instructions;        // Fixed size instructions TODO does
5435   max_instructions_per_bundle = 2;   // A53 = 2, A57 = 4
5436   // ARM instructions come in 32-bit word units
5437   instruction_unit_size = 4;         // An instruction is 4 bytes long
5438   instruction_fetch_unit_size = 64;  // The processor fetches one line
5439   instruction_fetch_units = 1;       // of 64 bytes
5440 
5441   // List of nop instructions
5442   nops( MachNop );
5443 %}
5444 
5445 // We don't use an actual pipeline model so don't care about resources
5446 // or description. we do use pipeline classes to introduce fixed
5447 // latencies
5448 
5449 //----------RESOURCES----------------------------------------------------------
5450 // Resources are the functional units available to the machine
5451 
5452 resources( INS0, INS1, INS01 = INS0 | INS1,
5453            ALU0, ALU1, ALU = ALU0 | ALU1,
5454            MAC,
5455            DIV,
5456            BRANCH,
5457            LDST,
5458            NEON_FP);
5459 
5460 //----------PIPELINE DESCRIPTION-----------------------------------------------
5461 // Pipeline Description specifies the stages in the machine's pipeline
5462 
5463 // Define the pipeline as a generic 6 stage pipeline
5464 pipe_desc(S0, S1, S2, S3, S4, S5);
5465 
5466 //----------PIPELINE CLASSES---------------------------------------------------
5467 // Pipeline Classes describe the stages in which input and output are
5468 // referenced by the hardware pipeline.
5469 
5470 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5471 %{
5472   single_instruction;
5473   src1   : S1(read);
5474   src2   : S2(read);
5475   dst    : S5(write);
5476   INS01  : ISS;
5477   NEON_FP : S5;
5478 %}
5479 
5480 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5481 %{
5482   single_instruction;
5483   src1   : S1(read);
5484   src2   : S2(read);
5485   dst    : S5(write);
5486   INS01  : ISS;
5487   NEON_FP : S5;
5488 %}
5489 
5490 pipe_class fp_uop_s(vRegF dst, vRegF src)
5491 %{
5492   single_instruction;
5493   src    : S1(read);
5494   dst    : S5(write);
5495   INS01  : ISS;
5496   NEON_FP : S5;
5497 %}
5498 
5499 pipe_class fp_uop_d(vRegD dst, vRegD src)
5500 %{
5501   single_instruction;
5502   src    : S1(read);
5503   dst    : S5(write);
5504   INS01  : ISS;
5505   NEON_FP : S5;
5506 %}
5507 
5508 pipe_class fp_d2f(vRegF dst, vRegD src)
5509 %{
5510   single_instruction;
5511   src    : S1(read);
5512   dst    : S5(write);
5513   INS01  : ISS;
5514   NEON_FP : S5;
5515 %}
5516 
5517 pipe_class fp_f2d(vRegD dst, vRegF src)
5518 %{
5519   single_instruction;
5520   src    : S1(read);
5521   dst    : S5(write);
5522   INS01  : ISS;
5523   NEON_FP : S5;
5524 %}
5525 
5526 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5527 %{
5528   single_instruction;
5529   src    : S1(read);
5530   dst    : S5(write);
5531   INS01  : ISS;
5532   NEON_FP : S5;
5533 %}
5534 
5535 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5536 %{
5537   single_instruction;
5538   src    : S1(read);
5539   dst    : S5(write);
5540   INS01  : ISS;
5541   NEON_FP : S5;
5542 %}
5543 
5544 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5545 %{
5546   single_instruction;
5547   src    : S1(read);
5548   dst    : S5(write);
5549   INS01  : ISS;
5550   NEON_FP : S5;
5551 %}
5552 
5553 pipe_class fp_l2f(vRegF dst, iRegL src)
5554 %{
5555   single_instruction;
5556   src    : S1(read);
5557   dst    : S5(write);
5558   INS01  : ISS;
5559   NEON_FP : S5;
5560 %}
5561 
5562 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5563 %{
5564   single_instruction;
5565   src    : S1(read);
5566   dst    : S5(write);
5567   INS01  : ISS;
5568   NEON_FP : S5;
5569 %}
5570 
5571 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5572 %{
5573   single_instruction;
5574   src    : S1(read);
5575   dst    : S5(write);
5576   INS01  : ISS;
5577   NEON_FP : S5;
5578 %}
5579 
5580 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5581 %{
5582   single_instruction;
5583   src    : S1(read);
5584   dst    : S5(write);
5585   INS01  : ISS;
5586   NEON_FP : S5;
5587 %}
5588 
5589 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5590 %{
5591   single_instruction;
5592   src    : S1(read);
5593   dst    : S5(write);
5594   INS01  : ISS;
5595   NEON_FP : S5;
5596 %}
5597 
5598 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5599 %{
5600   single_instruction;
5601   src1   : S1(read);
5602   src2   : S2(read);
5603   dst    : S5(write);
5604   INS0   : ISS;
5605   NEON_FP : S5;
5606 %}
5607 
5608 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5609 %{
5610   single_instruction;
5611   src1   : S1(read);
5612   src2   : S2(read);
5613   dst    : S5(write);
5614   INS0   : ISS;
5615   NEON_FP : S5;
5616 %}
5617 
5618 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5619 %{
5620   single_instruction;
5621   cr     : S1(read);
5622   src1   : S1(read);
5623   src2   : S1(read);
5624   dst    : S3(write);
5625   INS01  : ISS;
5626   NEON_FP : S3;
5627 %}
5628 
5629 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5630 %{
5631   single_instruction;
5632   cr     : S1(read);
5633   src1   : S1(read);
5634   src2   : S1(read);
5635   dst    : S3(write);
5636   INS01  : ISS;
5637   NEON_FP : S3;
5638 %}
5639 
5640 pipe_class fp_imm_s(vRegF dst)
5641 %{
5642   single_instruction;
5643   dst    : S3(write);
5644   INS01  : ISS;
5645   NEON_FP : S3;
5646 %}
5647 
5648 pipe_class fp_imm_d(vRegD dst)
5649 %{
5650   single_instruction;
5651   dst    : S3(write);
5652   INS01  : ISS;
5653   NEON_FP : S3;
5654 %}
5655 
5656 pipe_class fp_load_constant_s(vRegF dst)
5657 %{
5658   single_instruction;
5659   dst    : S4(write);
5660   INS01  : ISS;
5661   NEON_FP : S4;
5662 %}
5663 
5664 pipe_class fp_load_constant_d(vRegD dst)
5665 %{
5666   single_instruction;
5667   dst    : S4(write);
5668   INS01  : ISS;
5669   NEON_FP : S4;
5670 %}
5671 
5672 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
5673 %{
5674   single_instruction;
5675   dst    : S5(write);
5676   src1   : S1(read);
5677   src2   : S1(read);
5678   INS01  : ISS;
5679   NEON_FP : S5;
5680 %}
5681 
5682 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
5683 %{
5684   single_instruction;
5685   dst    : S5(write);
5686   src1   : S1(read);
5687   src2   : S1(read);
5688   INS0   : ISS;
5689   NEON_FP : S5;
5690 %}
5691 
5692 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
5693 %{
5694   single_instruction;
5695   dst    : S5(write);
5696   src1   : S1(read);
5697   src2   : S1(read);
5698   dst    : S1(read);
5699   INS01  : ISS;
5700   NEON_FP : S5;
5701 %}
5702 
5703 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
5704 %{
5705   single_instruction;
5706   dst    : S5(write);
5707   src1   : S1(read);
5708   src2   : S1(read);
5709   dst    : S1(read);
5710   INS0   : ISS;
5711   NEON_FP : S5;
5712 %}
5713 
5714 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
5715 %{
5716   single_instruction;
5717   dst    : S4(write);
5718   src1   : S2(read);
5719   src2   : S2(read);
5720   INS01  : ISS;
5721   NEON_FP : S4;
5722 %}
5723 
5724 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
5725 %{
5726   single_instruction;
5727   dst    : S4(write);
5728   src1   : S2(read);
5729   src2   : S2(read);
5730   INS0   : ISS;
5731   NEON_FP : S4;
5732 %}
5733 
5734 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
5735 %{
5736   single_instruction;
5737   dst    : S3(write);
5738   src1   : S2(read);
5739   src2   : S2(read);
5740   INS01  : ISS;
5741   NEON_FP : S3;
5742 %}
5743 
5744 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
5745 %{
5746   single_instruction;
5747   dst    : S3(write);
5748   src1   : S2(read);
5749   src2   : S2(read);
5750   INS0   : ISS;
5751   NEON_FP : S3;
5752 %}
5753 
5754 pipe_class vshift64(vecD dst, vecD src, vecX shift)
5755 %{
5756   single_instruction;
5757   dst    : S3(write);
5758   src    : S1(read);
5759   shift  : S1(read);
5760   INS01  : ISS;
5761   NEON_FP : S3;
5762 %}
5763 
5764 pipe_class vshift128(vecX dst, vecX src, vecX shift)
5765 %{
5766   single_instruction;
5767   dst    : S3(write);
5768   src    : S1(read);
5769   shift  : S1(read);
5770   INS0   : ISS;
5771   NEON_FP : S3;
5772 %}
5773 
5774 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
5775 %{
5776   single_instruction;
5777   dst    : S3(write);
5778   src    : S1(read);
5779   INS01  : ISS;
5780   NEON_FP : S3;
5781 %}
5782 
5783 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
5784 %{
5785   single_instruction;
5786   dst    : S3(write);
5787   src    : S1(read);
5788   INS0   : ISS;
5789   NEON_FP : S3;
5790 %}
5791 
5792 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
5793 %{
5794   single_instruction;
5795   dst    : S5(write);
5796   src1   : S1(read);
5797   src2   : S1(read);
5798   INS01  : ISS;
5799   NEON_FP : S5;
5800 %}
5801 
5802 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
5803 %{
5804   single_instruction;
5805   dst    : S5(write);
5806   src1   : S1(read);
5807   src2   : S1(read);
5808   INS0   : ISS;
5809   NEON_FP : S5;
5810 %}
5811 
5812 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
5813 %{
5814   single_instruction;
5815   dst    : S5(write);
5816   src1   : S1(read);
5817   src2   : S1(read);
5818   INS0   : ISS;
5819   NEON_FP : S5;
5820 %}
5821 
5822 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
5823 %{
5824   single_instruction;
5825   dst    : S5(write);
5826   src1   : S1(read);
5827   src2   : S1(read);
5828   INS0   : ISS;
5829   NEON_FP : S5;
5830 %}
5831 
5832 pipe_class vsqrt_fp128(vecX dst, vecX src)
5833 %{
5834   single_instruction;
5835   dst    : S5(write);
5836   src    : S1(read);
5837   INS0   : ISS;
5838   NEON_FP : S5;
5839 %}
5840 
5841 pipe_class vunop_fp64(vecD dst, vecD src)
5842 %{
5843   single_instruction;
5844   dst    : S5(write);
5845   src    : S1(read);
5846   INS01  : ISS;
5847   NEON_FP : S5;
5848 %}
5849 
5850 pipe_class vunop_fp128(vecX dst, vecX src)
5851 %{
5852   single_instruction;
5853   dst    : S5(write);
5854   src    : S1(read);
5855   INS0   : ISS;
5856   NEON_FP : S5;
5857 %}
5858 
5859 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
5860 %{
5861   single_instruction;
5862   dst    : S3(write);
5863   src    : S1(read);
5864   INS01  : ISS;
5865   NEON_FP : S3;
5866 %}
5867 
5868 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
5869 %{
5870   single_instruction;
5871   dst    : S3(write);
5872   src    : S1(read);
5873   INS01  : ISS;
5874   NEON_FP : S3;
5875 %}
5876 
5877 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
5878 %{
5879   single_instruction;
5880   dst    : S3(write);
5881   src    : S1(read);
5882   INS01  : ISS;
5883   NEON_FP : S3;
5884 %}
5885 
5886 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
5887 %{
5888   single_instruction;
5889   dst    : S3(write);
5890   src    : S1(read);
5891   INS01  : ISS;
5892   NEON_FP : S3;
5893 %}
5894 
5895 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
5896 %{
5897   single_instruction;
5898   dst    : S3(write);
5899   src    : S1(read);
5900   INS01  : ISS;
5901   NEON_FP : S3;
5902 %}
5903 
5904 pipe_class vmovi_reg_imm64(vecD dst)
5905 %{
5906   single_instruction;
5907   dst    : S3(write);
5908   INS01  : ISS;
5909   NEON_FP : S3;
5910 %}
5911 
5912 pipe_class vmovi_reg_imm128(vecX dst)
5913 %{
5914   single_instruction;
5915   dst    : S3(write);
5916   INS0   : ISS;
5917   NEON_FP : S3;
5918 %}
5919 
5920 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
5921 %{
5922   single_instruction;
5923   dst    : S5(write);
5924   mem    : ISS(read);
5925   INS01  : ISS;
5926   NEON_FP : S3;
5927 %}
5928 
5929 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
5930 %{
5931   single_instruction;
5932   dst    : S5(write);
5933   mem    : ISS(read);
5934   INS01  : ISS;
5935   NEON_FP : S3;
5936 %}
5937 
5938 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
5939 %{
5940   single_instruction;
5941   mem    : ISS(read);
5942   src    : S2(read);
5943   INS01  : ISS;
5944   NEON_FP : S3;
5945 %}
5946 
5947 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
5948 %{
5949   single_instruction;
5950   mem    : ISS(read);
5951   src    : S2(read);
5952   INS01  : ISS;
5953   NEON_FP : S3;
5954 %}
5955 
5956 //------- Integer ALU operations --------------------------
5957 
5958 // Integer ALU reg-reg operation
5959 // Operands needed in EX1, result generated in EX2
5960 // Eg.  ADD     x0, x1, x2
5961 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5962 %{
5963   single_instruction;
5964   dst    : EX2(write);
5965   src1   : EX1(read);
5966   src2   : EX1(read);
5967   INS01  : ISS; // Dual issue as instruction 0 or 1
5968   ALU    : EX2;
5969 %}
5970 
5971 // Integer ALU reg-reg operation with constant shift
5972 // Shifted register must be available in LATE_ISS instead of EX1
5973 // Eg.  ADD     x0, x1, x2, LSL #2
5974 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
5975 %{
5976   single_instruction;
5977   dst    : EX2(write);
5978   src1   : EX1(read);
5979   src2   : ISS(read);
5980   INS01  : ISS;
5981   ALU    : EX2;
5982 %}
5983 
5984 // Integer ALU reg operation with constant shift
5985 // Eg.  LSL     x0, x1, #shift
5986 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
5987 %{
5988   single_instruction;
5989   dst    : EX2(write);
5990   src1   : ISS(read);
5991   INS01  : ISS;
5992   ALU    : EX2;
5993 %}
5994 
5995 // Integer ALU reg-reg operation with variable shift
5996 // Both operands must be available in LATE_ISS instead of EX1
5997 // Result is available in EX1 instead of EX2
5998 // Eg.  LSLV    x0, x1, x2
5999 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6000 %{
6001   single_instruction;
6002   dst    : EX1(write);
6003   src1   : ISS(read);
6004   src2   : ISS(read);
6005   INS01  : ISS;
6006   ALU    : EX1;
6007 %}
6008 
6009 // Integer ALU reg-reg operation with extract
6010 // As for _vshift above, but result generated in EX2
6011 // Eg.  EXTR    x0, x1, x2, #N
6012 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6013 %{
6014   single_instruction;
6015   dst    : EX2(write);
6016   src1   : ISS(read);
6017   src2   : ISS(read);
6018   INS1   : ISS; // Can only dual issue as Instruction 1
6019   ALU    : EX1;
6020 %}
6021 
6022 // Integer ALU reg operation
6023 // Eg.  NEG     x0, x1
6024 pipe_class ialu_reg(iRegI dst, iRegI src)
6025 %{
6026   single_instruction;
6027   dst    : EX2(write);
6028   src    : EX1(read);
6029   INS01  : ISS;
6030   ALU    : EX2;
6031 %}
6032 
6033 // Integer ALU reg mmediate operation
6034 // Eg.  ADD     x0, x1, #N
6035 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6036 %{
6037   single_instruction;
6038   dst    : EX2(write);
6039   src1   : EX1(read);
6040   INS01  : ISS;
6041   ALU    : EX2;
6042 %}
6043 
6044 // Integer ALU immediate operation (no source operands)
6045 // Eg.  MOV     x0, #N
6046 pipe_class ialu_imm(iRegI dst)
6047 %{
6048   single_instruction;
6049   dst    : EX1(write);
6050   INS01  : ISS;
6051   ALU    : EX1;
6052 %}
6053 
6054 //------- Compare operation -------------------------------
6055 
6056 // Compare reg-reg
6057 // Eg.  CMP     x0, x1
6058 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6059 %{
6060   single_instruction;
6061 //  fixed_latency(16);
6062   cr     : EX2(write);
6063   op1    : EX1(read);
6064   op2    : EX1(read);
6065   INS01  : ISS;
6066   ALU    : EX2;
6067 %}
6068 
6069 // Compare reg-reg
6070 // Eg.  CMP     x0, #N
6071 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6072 %{
6073   single_instruction;
6074 //  fixed_latency(16);
6075   cr     : EX2(write);
6076   op1    : EX1(read);
6077   INS01  : ISS;
6078   ALU    : EX2;
6079 %}
6080 
6081 //------- Conditional instructions ------------------------
6082 
6083 // Conditional no operands
6084 // Eg.  CSINC   x0, zr, zr, <cond>
6085 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6086 %{
6087   single_instruction;
6088   cr     : EX1(read);
6089   dst    : EX2(write);
6090   INS01  : ISS;
6091   ALU    : EX2;
6092 %}
6093 
6094 // Conditional 2 operand
6095 // EG.  CSEL    X0, X1, X2, <cond>
6096 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6097 %{
6098   single_instruction;
6099   cr     : EX1(read);
6100   src1   : EX1(read);
6101   src2   : EX1(read);
6102   dst    : EX2(write);
6103   INS01  : ISS;
6104   ALU    : EX2;
6105 %}
6106 
6107 // Conditional 2 operand
6108 // EG.  CSEL    X0, X1, X2, <cond>
6109 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6110 %{
6111   single_instruction;
6112   cr     : EX1(read);
6113   src    : EX1(read);
6114   dst    : EX2(write);
6115   INS01  : ISS;
6116   ALU    : EX2;
6117 %}
6118 
6119 //------- Multiply pipeline operations --------------------
6120 
6121 // Multiply reg-reg
6122 // Eg.  MUL     w0, w1, w2
6123 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6124 %{
6125   single_instruction;
6126   dst    : WR(write);
6127   src1   : ISS(read);
6128   src2   : ISS(read);
6129   INS01  : ISS;
6130   MAC    : WR;
6131 %}
6132 
6133 // Multiply accumulate
6134 // Eg.  MADD    w0, w1, w2, w3
6135 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6136 %{
6137   single_instruction;
6138   dst    : WR(write);
6139   src1   : ISS(read);
6140   src2   : ISS(read);
6141   src3   : ISS(read);
6142   INS01  : ISS;
6143   MAC    : WR;
6144 %}
6145 
6146 // Eg.  MUL     w0, w1, w2
6147 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6148 %{
6149   single_instruction;
6150   fixed_latency(3); // Maximum latency for 64 bit mul
6151   dst    : WR(write);
6152   src1   : ISS(read);
6153   src2   : ISS(read);
6154   INS01  : ISS;
6155   MAC    : WR;
6156 %}
6157 
6158 // Multiply accumulate
6159 // Eg.  MADD    w0, w1, w2, w3
6160 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6161 %{
6162   single_instruction;
6163   fixed_latency(3); // Maximum latency for 64 bit mul
6164   dst    : WR(write);
6165   src1   : ISS(read);
6166   src2   : ISS(read);
6167   src3   : ISS(read);
6168   INS01  : ISS;
6169   MAC    : WR;
6170 %}
6171 
6172 //------- Divide pipeline operations --------------------
6173 
6174 // Eg.  SDIV    w0, w1, w2
6175 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6176 %{
6177   single_instruction;
6178   fixed_latency(8); // Maximum latency for 32 bit divide
6179   dst    : WR(write);
6180   src1   : ISS(read);
6181   src2   : ISS(read);
6182   INS0   : ISS; // Can only dual issue as instruction 0
6183   DIV    : WR;
6184 %}
6185 
6186 // Eg.  SDIV    x0, x1, x2
6187 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6188 %{
6189   single_instruction;
6190   fixed_latency(16); // Maximum latency for 64 bit divide
6191   dst    : WR(write);
6192   src1   : ISS(read);
6193   src2   : ISS(read);
6194   INS0   : ISS; // Can only dual issue as instruction 0
6195   DIV    : WR;
6196 %}
6197 
6198 //------- Load pipeline operations ------------------------
6199 
6200 // Load - prefetch
6201 // Eg.  PFRM    <mem>
6202 pipe_class iload_prefetch(memory mem)
6203 %{
6204   single_instruction;
6205   mem    : ISS(read);
6206   INS01  : ISS;
6207   LDST   : WR;
6208 %}
6209 
6210 // Load - reg, mem
6211 // Eg.  LDR     x0, <mem>
6212 pipe_class iload_reg_mem(iRegI dst, memory mem)
6213 %{
6214   single_instruction;
6215   dst    : WR(write);
6216   mem    : ISS(read);
6217   INS01  : ISS;
6218   LDST   : WR;
6219 %}
6220 
6221 // Load - reg, reg
6222 // Eg.  LDR     x0, [sp, x1]
6223 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6224 %{
6225   single_instruction;
6226   dst    : WR(write);
6227   src    : ISS(read);
6228   INS01  : ISS;
6229   LDST   : WR;
6230 %}
6231 
6232 //------- Store pipeline operations -----------------------
6233 
6234 // Store - zr, mem
6235 // Eg.  STR     zr, <mem>
6236 pipe_class istore_mem(memory mem)
6237 %{
6238   single_instruction;
6239   mem    : ISS(read);
6240   INS01  : ISS;
6241   LDST   : WR;
6242 %}
6243 
6244 // Store - reg, mem
6245 // Eg.  STR     x0, <mem>
6246 pipe_class istore_reg_mem(iRegI src, memory mem)
6247 %{
6248   single_instruction;
6249   mem    : ISS(read);
6250   src    : EX2(read);
6251   INS01  : ISS;
6252   LDST   : WR;
6253 %}
6254 
6255 // Store - reg, reg
6256 // Eg. STR      x0, [sp, x1]
6257 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6258 %{
6259   single_instruction;
6260   dst    : ISS(read);
6261   src    : EX2(read);
6262   INS01  : ISS;
6263   LDST   : WR;
6264 %}
6265 
6266 //------- Store pipeline operations -----------------------
6267 
6268 // Branch
6269 pipe_class pipe_branch()
6270 %{
6271   single_instruction;
6272   INS01  : ISS;
6273   BRANCH : EX1;
6274 %}
6275 
6276 // Conditional branch
6277 pipe_class pipe_branch_cond(rFlagsReg cr)
6278 %{
6279   single_instruction;
6280   cr     : EX1(read);
6281   INS01  : ISS;
6282   BRANCH : EX1;
6283 %}
6284 
6285 // Compare & Branch
6286 // EG.  CBZ/CBNZ
6287 pipe_class pipe_cmp_branch(iRegI op1)
6288 %{
6289   single_instruction;
6290   op1    : EX1(read);
6291   INS01  : ISS;
6292   BRANCH : EX1;
6293 %}
6294 
6295 //------- Synchronisation operations ----------------------
6296 
6297 // Any operation requiring serialization.
6298 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
6299 pipe_class pipe_serial()
6300 %{
6301   single_instruction;
6302   force_serialization;
6303   fixed_latency(16);
6304   INS01  : ISS(2); // Cannot dual issue with any other instruction
6305   LDST   : WR;
6306 %}
6307 
6308 // Generic big/slow expanded idiom - also serialized
6309 pipe_class pipe_slow()
6310 %{
6311   instruction_count(10);
6312   multiple_bundles;
6313   force_serialization;
6314   fixed_latency(16);
6315   INS01  : ISS(2); // Cannot dual issue with any other instruction
6316   LDST   : WR;
6317 %}
6318 
6319 // Empty pipeline class
6320 pipe_class pipe_class_empty()
6321 %{
6322   single_instruction;
6323   fixed_latency(0);
6324 %}
6325 
6326 // Default pipeline class.
6327 pipe_class pipe_class_default()
6328 %{
6329   single_instruction;
6330   fixed_latency(2);
6331 %}
6332 
6333 // Pipeline class for compares.
6334 pipe_class pipe_class_compare()
6335 %{
6336   single_instruction;
6337   fixed_latency(16);
6338 %}
6339 
6340 // Pipeline class for memory operations.
6341 pipe_class pipe_class_memory()
6342 %{
6343   single_instruction;
6344   fixed_latency(16);
6345 %}
6346 
6347 // Pipeline class for call.
6348 pipe_class pipe_class_call()
6349 %{
6350   single_instruction;
6351   fixed_latency(100);
6352 %}
6353 
6354 // Define the class for the Nop node.
6355 define %{
6356    MachNop = pipe_class_empty;
6357 %}
6358 
6359 %}
6360 //----------INSTRUCTIONS-------------------------------------------------------
6361 //
6362 // match      -- States which machine-independent subtree may be replaced
6363 //               by this instruction.
6364 // ins_cost   -- The estimated cost of this instruction is used by instruction
6365 //               selection to identify a minimum cost tree of machine
6366 //               instructions that matches a tree of machine-independent
6367 //               instructions.
6368 // format     -- A string providing the disassembly for this instruction.
6369 //               The value of an instruction's operand may be inserted
6370 //               by referring to it with a '$' prefix.
6371 // opcode     -- Three instruction opcodes may be provided.  These are referred
6372 //               to within an encode class as $primary, $secondary, and $tertiary
6373 //               rrspectively.  The primary opcode is commonly used to
6374 //               indicate the type of machine instruction, while secondary
6375 //               and tertiary are often used for prefix options or addressing
6376 //               modes.
6377 // ins_encode -- A list of encode classes with parameters. The encode class
6378 //               name must have been defined in an 'enc_class' specification
6379 //               in the encode section of the architecture description.
6380 
6381 // ============================================================================
6382 // Memory (Load/Store) Instructions
6383 
6384 // Load Instructions
6385 
6386 // Load Byte (8 bit signed)
6387 instruct loadB(iRegINoSp dst, memory mem)
6388 %{
6389   match(Set dst (LoadB mem));
6390   predicate(!needs_acquiring_load(n));
6391 
6392   ins_cost(4 * INSN_COST);
6393   format %{ "ldrsbw  $dst, $mem\t# byte" %}
6394 
6395   ins_encode(aarch64_enc_ldrsbw(dst, mem));
6396 
6397   ins_pipe(iload_reg_mem);
6398 %}
6399 
6400 // Load Byte (8 bit signed) into long
6401 instruct loadB2L(iRegLNoSp dst, memory mem)
6402 %{
6403   match(Set dst (ConvI2L (LoadB mem)));
6404   predicate(!needs_acquiring_load(n->in(1)));
6405 
6406   ins_cost(4 * INSN_COST);
6407   format %{ "ldrsb  $dst, $mem\t# byte" %}
6408 
6409   ins_encode(aarch64_enc_ldrsb(dst, mem));
6410 
6411   ins_pipe(iload_reg_mem);
6412 %}
6413 
6414 // Load Byte (8 bit unsigned)
6415 instruct loadUB(iRegINoSp dst, memory mem)
6416 %{
6417   match(Set dst (LoadUB mem));
6418   predicate(!needs_acquiring_load(n));
6419 
6420   ins_cost(4 * INSN_COST);
6421   format %{ "ldrbw  $dst, $mem\t# byte" %}
6422 
6423   ins_encode(aarch64_enc_ldrb(dst, mem));
6424 
6425   ins_pipe(iload_reg_mem);
6426 %}
6427 
6428 // Load Byte (8 bit unsigned) into long
6429 instruct loadUB2L(iRegLNoSp dst, memory mem)
6430 %{
6431   match(Set dst (ConvI2L (LoadUB mem)));
6432   predicate(!needs_acquiring_load(n->in(1)));
6433 
6434   ins_cost(4 * INSN_COST);
6435   format %{ "ldrb  $dst, $mem\t# byte" %}
6436 
6437   ins_encode(aarch64_enc_ldrb(dst, mem));
6438 
6439   ins_pipe(iload_reg_mem);
6440 %}
6441 
6442 // Load Short (16 bit signed)
6443 instruct loadS(iRegINoSp dst, memory mem)
6444 %{
6445   match(Set dst (LoadS mem));
6446   predicate(!needs_acquiring_load(n));
6447 
6448   ins_cost(4 * INSN_COST);
6449   format %{ "ldrshw  $dst, $mem\t# short" %}
6450 
6451   ins_encode(aarch64_enc_ldrshw(dst, mem));
6452 
6453   ins_pipe(iload_reg_mem);
6454 %}
6455 
6456 // Load Short (16 bit signed) into long
6457 instruct loadS2L(iRegLNoSp dst, memory mem)
6458 %{
6459   match(Set dst (ConvI2L (LoadS mem)));
6460   predicate(!needs_acquiring_load(n->in(1)));
6461 
6462   ins_cost(4 * INSN_COST);
6463   format %{ "ldrsh  $dst, $mem\t# short" %}
6464 
6465   ins_encode(aarch64_enc_ldrsh(dst, mem));
6466 
6467   ins_pipe(iload_reg_mem);
6468 %}
6469 
6470 // Load Char (16 bit unsigned)
6471 instruct loadUS(iRegINoSp dst, memory mem)
6472 %{
6473   match(Set dst (LoadUS mem));
6474   predicate(!needs_acquiring_load(n));
6475 
6476   ins_cost(4 * INSN_COST);
6477   format %{ "ldrh  $dst, $mem\t# short" %}
6478 
6479   ins_encode(aarch64_enc_ldrh(dst, mem));
6480 
6481   ins_pipe(iload_reg_mem);
6482 %}
6483 
6484 // Load Short/Char (16 bit unsigned) into long
6485 instruct loadUS2L(iRegLNoSp dst, memory mem)
6486 %{
6487   match(Set dst (ConvI2L (LoadUS mem)));
6488   predicate(!needs_acquiring_load(n->in(1)));
6489 
6490   ins_cost(4 * INSN_COST);
6491   format %{ "ldrh  $dst, $mem\t# short" %}
6492 
6493   ins_encode(aarch64_enc_ldrh(dst, mem));
6494 
6495   ins_pipe(iload_reg_mem);
6496 %}
6497 
6498 // Load Integer (32 bit signed)
6499 instruct loadI(iRegINoSp dst, memory mem)
6500 %{
6501   match(Set dst (LoadI mem));
6502   predicate(!needs_acquiring_load(n));
6503 
6504   ins_cost(4 * INSN_COST);
6505   format %{ "ldrw  $dst, $mem\t# int" %}
6506 
6507   ins_encode(aarch64_enc_ldrw(dst, mem));
6508 
6509   ins_pipe(iload_reg_mem);
6510 %}
6511 
6512 // Load Integer (32 bit signed) into long
6513 instruct loadI2L(iRegLNoSp dst, memory mem)
6514 %{
6515   match(Set dst (ConvI2L (LoadI mem)));
6516   predicate(!needs_acquiring_load(n->in(1)));
6517 
6518   ins_cost(4 * INSN_COST);
6519   format %{ "ldrsw  $dst, $mem\t# int" %}
6520 
6521   ins_encode(aarch64_enc_ldrsw(dst, mem));
6522 
6523   ins_pipe(iload_reg_mem);
6524 %}
6525 
6526 // Load Integer (32 bit unsigned) into long
6527 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
6528 %{
6529   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6530   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6531 
6532   ins_cost(4 * INSN_COST);
6533   format %{ "ldrw  $dst, $mem\t# int" %}
6534 
6535   ins_encode(aarch64_enc_ldrw(dst, mem));
6536 
6537   ins_pipe(iload_reg_mem);
6538 %}
6539 
6540 // Load Long (64 bit signed)
6541 instruct loadL(iRegLNoSp dst, memory mem)
6542 %{
6543   match(Set dst (LoadL mem));
6544   predicate(!needs_acquiring_load(n));
6545 
6546   ins_cost(4 * INSN_COST);
6547   format %{ "ldr  $dst, $mem\t# int" %}
6548 
6549   ins_encode(aarch64_enc_ldr(dst, mem));
6550 
6551   ins_pipe(iload_reg_mem);
6552 %}
6553 
6554 // Load Range
6555 instruct loadRange(iRegINoSp dst, memory mem)
6556 %{
6557   match(Set dst (LoadRange mem));
6558 
6559   ins_cost(4 * INSN_COST);
6560   format %{ "ldrw  $dst, $mem\t# range" %}
6561 
6562   ins_encode(aarch64_enc_ldrw(dst, mem));
6563 
6564   ins_pipe(iload_reg_mem);
6565 %}
6566 
6567 // Load Pointer
6568 instruct loadP(iRegPNoSp dst, memory mem)
6569 %{
6570   match(Set dst (LoadP mem));
6571   predicate(!needs_acquiring_load(n));
6572 
6573   ins_cost(4 * INSN_COST);
6574   format %{ "ldr  $dst, $mem\t# ptr" %}
6575 
6576   ins_encode(aarch64_enc_ldr(dst, mem));
6577 
6578   ins_pipe(iload_reg_mem);
6579 %}
6580 
6581 // Load Compressed Pointer
6582 instruct loadN(iRegNNoSp dst, memory mem)
6583 %{
6584   match(Set dst (LoadN mem));
6585   predicate(!needs_acquiring_load(n));
6586 
6587   ins_cost(4 * INSN_COST);
6588   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
6589 
6590   ins_encode(aarch64_enc_ldrw(dst, mem));
6591 
6592   ins_pipe(iload_reg_mem);
6593 %}
6594 
6595 // Load Klass Pointer
6596 instruct loadKlass(iRegPNoSp dst, memory mem)
6597 %{
6598   match(Set dst (LoadKlass mem));
6599   predicate(!needs_acquiring_load(n));
6600 
6601   ins_cost(4 * INSN_COST);
6602   format %{ "ldr  $dst, $mem\t# class" %}
6603 
6604   ins_encode(aarch64_enc_ldr(dst, mem));
6605 
6606   ins_pipe(iload_reg_mem);
6607 %}
6608 
6609 // Load Narrow Klass Pointer
6610 instruct loadNKlass(iRegNNoSp dst, memory mem)
6611 %{
6612   match(Set dst (LoadNKlass mem));
6613   predicate(!needs_acquiring_load(n));
6614 
6615   ins_cost(4 * INSN_COST);
6616   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
6617 
6618   ins_encode(aarch64_enc_ldrw(dst, mem));
6619 
6620   ins_pipe(iload_reg_mem);
6621 %}
6622 
6623 // Load Float
6624 instruct loadF(vRegF dst, memory mem)
6625 %{
6626   match(Set dst (LoadF mem));
6627   predicate(!needs_acquiring_load(n));
6628 
6629   ins_cost(4 * INSN_COST);
6630   format %{ "ldrs  $dst, $mem\t# float" %}
6631 
6632   ins_encode( aarch64_enc_ldrs(dst, mem) );
6633 
6634   ins_pipe(pipe_class_memory);
6635 %}
6636 
6637 // Load Double
6638 instruct loadD(vRegD dst, memory mem)
6639 %{
6640   match(Set dst (LoadD mem));
6641   predicate(!needs_acquiring_load(n));
6642 
6643   ins_cost(4 * INSN_COST);
6644   format %{ "ldrd  $dst, $mem\t# double" %}
6645 
6646   ins_encode( aarch64_enc_ldrd(dst, mem) );
6647 
6648   ins_pipe(pipe_class_memory);
6649 %}
6650 
6651 
6652 // Load Int Constant
6653 instruct loadConI(iRegINoSp dst, immI src)
6654 %{
6655   match(Set dst src);
6656 
6657   ins_cost(INSN_COST);
6658   format %{ "mov $dst, $src\t# int" %}
6659 
6660   ins_encode( aarch64_enc_movw_imm(dst, src) );
6661 
6662   ins_pipe(ialu_imm);
6663 %}
6664 
6665 // Load Long Constant
6666 instruct loadConL(iRegLNoSp dst, immL src)
6667 %{
6668   match(Set dst src);
6669 
6670   ins_cost(INSN_COST);
6671   format %{ "mov $dst, $src\t# long" %}
6672 
6673   ins_encode( aarch64_enc_mov_imm(dst, src) );
6674 
6675   ins_pipe(ialu_imm);
6676 %}
6677 
6678 // Load Pointer Constant
6679 
6680 instruct loadConP(iRegPNoSp dst, immP con)
6681 %{
6682   match(Set dst con);
6683 
6684   ins_cost(INSN_COST * 4);
6685   format %{
6686     "mov  $dst, $con\t# ptr\n\t"
6687   %}
6688 
6689   ins_encode(aarch64_enc_mov_p(dst, con));
6690 
6691   ins_pipe(ialu_imm);
6692 %}
6693 
6694 // Load Null Pointer Constant
6695 
6696 instruct loadConP0(iRegPNoSp dst, immP0 con)
6697 %{
6698   match(Set dst con);
6699 
6700   ins_cost(INSN_COST);
6701   format %{ "mov  $dst, $con\t# NULL ptr" %}
6702 
6703   ins_encode(aarch64_enc_mov_p0(dst, con));
6704 
6705   ins_pipe(ialu_imm);
6706 %}
6707 
6708 // Load Pointer Constant One
6709 
6710 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6711 %{
6712   match(Set dst con);
6713 
6714   ins_cost(INSN_COST);
6715   format %{ "mov  $dst, $con\t# NULL ptr" %}
6716 
6717   ins_encode(aarch64_enc_mov_p1(dst, con));
6718 
6719   ins_pipe(ialu_imm);
6720 %}
6721 
6722 // Load Poll Page Constant
6723 
6724 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
6725 %{
6726   match(Set dst con);
6727 
6728   ins_cost(INSN_COST);
6729   format %{ "adr  $dst, $con\t# Poll Page Ptr" %}
6730 
6731   ins_encode(aarch64_enc_mov_poll_page(dst, con));
6732 
6733   ins_pipe(ialu_imm);
6734 %}
6735 
6736 // Load Byte Map Base Constant
6737 
6738 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6739 %{
6740   match(Set dst con);
6741 
6742   ins_cost(INSN_COST);
6743   format %{ "adr  $dst, $con\t# Byte Map Base" %}
6744 
6745   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6746 
6747   ins_pipe(ialu_imm);
6748 %}
6749 
6750 // Load Narrow Pointer Constant
6751 
6752 instruct loadConN(iRegNNoSp dst, immN con)
6753 %{
6754   match(Set dst con);
6755 
6756   ins_cost(INSN_COST * 4);
6757   format %{ "mov  $dst, $con\t# compressed ptr" %}
6758 
6759   ins_encode(aarch64_enc_mov_n(dst, con));
6760 
6761   ins_pipe(ialu_imm);
6762 %}
6763 
6764 // Load Narrow Null Pointer Constant
6765 
6766 instruct loadConN0(iRegNNoSp dst, immN0 con)
6767 %{
6768   match(Set dst con);
6769 
6770   ins_cost(INSN_COST);
6771   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
6772 
6773   ins_encode(aarch64_enc_mov_n0(dst, con));
6774 
6775   ins_pipe(ialu_imm);
6776 %}
6777 
6778 // Load Narrow Klass Constant
6779 
6780 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6781 %{
6782   match(Set dst con);
6783 
6784   ins_cost(INSN_COST);
6785   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
6786 
6787   ins_encode(aarch64_enc_mov_nk(dst, con));
6788 
6789   ins_pipe(ialu_imm);
6790 %}
6791 
6792 // Load Packed Float Constant
6793 
6794 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6795   match(Set dst con);
6796   ins_cost(INSN_COST * 4);
6797   format %{ "fmovs  $dst, $con"%}
6798   ins_encode %{
6799     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6800   %}
6801 
6802   ins_pipe(fp_imm_s);
6803 %}
6804 
6805 // Load Float Constant
6806 
6807 instruct loadConF(vRegF dst, immF con) %{
6808   match(Set dst con);
6809 
6810   ins_cost(INSN_COST * 4);
6811 
6812   format %{
6813     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6814   %}
6815 
6816   ins_encode %{
6817     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6818   %}
6819 
6820   ins_pipe(fp_load_constant_s);
6821 %}
6822 
6823 // Load Packed Double Constant
6824 
6825 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6826   match(Set dst con);
6827   ins_cost(INSN_COST);
6828   format %{ "fmovd  $dst, $con"%}
6829   ins_encode %{
6830     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6831   %}
6832 
6833   ins_pipe(fp_imm_d);
6834 %}
6835 
6836 // Load Double Constant
6837 
6838 instruct loadConD(vRegD dst, immD con) %{
6839   match(Set dst con);
6840 
6841   ins_cost(INSN_COST * 5);
6842   format %{
6843     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6844   %}
6845 
6846   ins_encode %{
6847     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6848   %}
6849 
6850   ins_pipe(fp_load_constant_d);
6851 %}
6852 
6853 // Store Instructions
6854 
6855 // Store CMS card-mark Immediate
6856 instruct storeimmCM0(immI0 zero, memory mem)
6857 %{
6858   match(Set mem (StoreCM mem zero));
6859   predicate(unnecessary_storestore(n));
6860 
6861   ins_cost(INSN_COST);
6862   format %{ "storestore (elided)\n\t"
6863             "strb zr, $mem\t# byte" %}
6864 
6865   ins_encode(aarch64_enc_strb0(mem));
6866 
6867   ins_pipe(istore_mem);
6868 %}
6869 
6870 // Store CMS card-mark Immediate with intervening StoreStore
6871 // needed when using CMS with no conditional card marking
6872 instruct storeimmCM0_ordered(immI0 zero, memory mem)
6873 %{
6874   match(Set mem (StoreCM mem zero));
6875 
6876   ins_cost(INSN_COST * 2);
6877   format %{ "storestore\n\t"
6878             "dmb ishst"
6879             "\n\tstrb zr, $mem\t# byte" %}
6880 
6881   ins_encode(aarch64_enc_strb0_ordered(mem));
6882 
6883   ins_pipe(istore_mem);
6884 %}
6885 
6886 // Store Byte
6887 instruct storeB(iRegIorL2I src, memory mem)
6888 %{
6889   match(Set mem (StoreB mem src));
6890   predicate(!needs_releasing_store(n));
6891 
6892   ins_cost(INSN_COST);
6893   format %{ "strb  $src, $mem\t# byte" %}
6894 
6895   ins_encode(aarch64_enc_strb(src, mem));
6896 
6897   ins_pipe(istore_reg_mem);
6898 %}
6899 
6900 
6901 instruct storeimmB0(immI0 zero, memory mem)
6902 %{
6903   match(Set mem (StoreB mem zero));
6904   predicate(!needs_releasing_store(n));
6905 
6906   ins_cost(INSN_COST);
6907   format %{ "strb rscractch2, $mem\t# byte" %}
6908 
6909   ins_encode(aarch64_enc_strb0(mem));
6910 
6911   ins_pipe(istore_mem);
6912 %}
6913 
6914 // Store Char/Short
6915 instruct storeC(iRegIorL2I src, memory mem)
6916 %{
6917   match(Set mem (StoreC mem src));
6918   predicate(!needs_releasing_store(n));
6919 
6920   ins_cost(INSN_COST);
6921   format %{ "strh  $src, $mem\t# short" %}
6922 
6923   ins_encode(aarch64_enc_strh(src, mem));
6924 
6925   ins_pipe(istore_reg_mem);
6926 %}
6927 
6928 instruct storeimmC0(immI0 zero, memory mem)
6929 %{
6930   match(Set mem (StoreC mem zero));
6931   predicate(!needs_releasing_store(n));
6932 
6933   ins_cost(INSN_COST);
6934   format %{ "strh  zr, $mem\t# short" %}
6935 
6936   ins_encode(aarch64_enc_strh0(mem));
6937 
6938   ins_pipe(istore_mem);
6939 %}
6940 
6941 // Store Integer
6942 
6943 instruct storeI(iRegIorL2I src, memory mem)
6944 %{
6945   match(Set mem(StoreI mem src));
6946   predicate(!needs_releasing_store(n));
6947 
6948   ins_cost(INSN_COST);
6949   format %{ "strw  $src, $mem\t# int" %}
6950 
6951   ins_encode(aarch64_enc_strw(src, mem));
6952 
6953   ins_pipe(istore_reg_mem);
6954 %}
6955 
6956 instruct storeimmI0(immI0 zero, memory mem)
6957 %{
6958   match(Set mem(StoreI mem zero));
6959   predicate(!needs_releasing_store(n));
6960 
6961   ins_cost(INSN_COST);
6962   format %{ "strw  zr, $mem\t# int" %}
6963 
6964   ins_encode(aarch64_enc_strw0(mem));
6965 
6966   ins_pipe(istore_mem);
6967 %}
6968 
6969 // Store Long (64 bit signed)
6970 instruct storeL(iRegL src, memory mem)
6971 %{
6972   match(Set mem (StoreL mem src));
6973   predicate(!needs_releasing_store(n));
6974 
6975   ins_cost(INSN_COST);
6976   format %{ "str  $src, $mem\t# int" %}
6977 
6978   ins_encode(aarch64_enc_str(src, mem));
6979 
6980   ins_pipe(istore_reg_mem);
6981 %}
6982 
6983 // Store Long (64 bit signed)
6984 instruct storeimmL0(immL0 zero, memory mem)
6985 %{
6986   match(Set mem (StoreL mem zero));
6987   predicate(!needs_releasing_store(n));
6988 
6989   ins_cost(INSN_COST);
6990   format %{ "str  zr, $mem\t# int" %}
6991 
6992   ins_encode(aarch64_enc_str0(mem));
6993 
6994   ins_pipe(istore_mem);
6995 %}
6996 
6997 // Store Pointer
6998 instruct storeP(iRegP src, memory mem)
6999 %{
7000   match(Set mem (StoreP mem src));
7001   predicate(!needs_releasing_store(n));
7002 
7003   ins_cost(INSN_COST);
7004   format %{ "str  $src, $mem\t# ptr" %}
7005 
7006   ins_encode(aarch64_enc_str(src, mem));
7007 
7008   ins_pipe(istore_reg_mem);
7009 %}
7010 
7011 // Store Pointer
7012 instruct storeimmP0(immP0 zero, memory mem)
7013 %{
7014   match(Set mem (StoreP mem zero));
7015   predicate(!needs_releasing_store(n));
7016 
7017   ins_cost(INSN_COST);
7018   format %{ "str zr, $mem\t# ptr" %}
7019 
7020   ins_encode(aarch64_enc_str0(mem));
7021 
7022   ins_pipe(istore_mem);
7023 %}
7024 
7025 // Store Compressed Pointer
7026 instruct storeN(iRegN src, memory mem)
7027 %{
7028   match(Set mem (StoreN mem src));
7029   predicate(!needs_releasing_store(n));
7030 
7031   ins_cost(INSN_COST);
7032   format %{ "strw  $src, $mem\t# compressed ptr" %}
7033 
7034   ins_encode(aarch64_enc_strw(src, mem));
7035 
7036   ins_pipe(istore_reg_mem);
7037 %}
7038 
7039 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
7040 %{
7041   match(Set mem (StoreN mem zero));
7042   predicate(CompressedOops::base() == NULL &&
7043             CompressedKlassPointers::base() == NULL &&
7044             (!needs_releasing_store(n)));
7045 
7046   ins_cost(INSN_COST);
7047   format %{ "strw  rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
7048 
7049   ins_encode(aarch64_enc_strw(heapbase, mem));
7050 
7051   ins_pipe(istore_reg_mem);
7052 %}
7053 
7054 // Store Float
7055 instruct storeF(vRegF src, memory mem)
7056 %{
7057   match(Set mem (StoreF mem src));
7058   predicate(!needs_releasing_store(n));
7059 
7060   ins_cost(INSN_COST);
7061   format %{ "strs  $src, $mem\t# float" %}
7062 
7063   ins_encode( aarch64_enc_strs(src, mem) );
7064 
7065   ins_pipe(pipe_class_memory);
7066 %}
7067 
7068 // TODO
7069 // implement storeImmF0 and storeFImmPacked
7070 
7071 // Store Double
7072 instruct storeD(vRegD src, memory mem)
7073 %{
7074   match(Set mem (StoreD mem src));
7075   predicate(!needs_releasing_store(n));
7076 
7077   ins_cost(INSN_COST);
7078   format %{ "strd  $src, $mem\t# double" %}
7079 
7080   ins_encode( aarch64_enc_strd(src, mem) );
7081 
7082   ins_pipe(pipe_class_memory);
7083 %}
7084 
7085 // Store Compressed Klass Pointer
7086 instruct storeNKlass(iRegN src, memory mem)
7087 %{
7088   predicate(!needs_releasing_store(n));
7089   match(Set mem (StoreNKlass mem src));
7090 
7091   ins_cost(INSN_COST);
7092   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
7093 
7094   ins_encode(aarch64_enc_strw(src, mem));
7095 
7096   ins_pipe(istore_reg_mem);
7097 %}
7098 
7099 // TODO
7100 // implement storeImmD0 and storeDImmPacked
7101 
7102 // prefetch instructions
7103 // Must be safe to execute with invalid address (cannot fault).
7104 
7105 instruct prefetchalloc( memory mem ) %{
7106   match(PrefetchAllocation mem);
7107 
7108   ins_cost(INSN_COST);
7109   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7110 
7111   ins_encode( aarch64_enc_prefetchw(mem) );
7112 
7113   ins_pipe(iload_prefetch);
7114 %}
7115 
7116 //  ---------------- volatile loads and stores ----------------
7117 
7118 // Load Byte (8 bit signed)
7119 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7120 %{
7121   match(Set dst (LoadB mem));
7122 
7123   ins_cost(VOLATILE_REF_COST);
7124   format %{ "ldarsb  $dst, $mem\t# byte" %}
7125 
7126   ins_encode(aarch64_enc_ldarsb(dst, mem));
7127 
7128   ins_pipe(pipe_serial);
7129 %}
7130 
7131 // Load Byte (8 bit signed) into long
7132 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7133 %{
7134   match(Set dst (ConvI2L (LoadB mem)));
7135 
7136   ins_cost(VOLATILE_REF_COST);
7137   format %{ "ldarsb  $dst, $mem\t# byte" %}
7138 
7139   ins_encode(aarch64_enc_ldarsb(dst, mem));
7140 
7141   ins_pipe(pipe_serial);
7142 %}
7143 
7144 // Load Byte (8 bit unsigned)
7145 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7146 %{
7147   match(Set dst (LoadUB mem));
7148 
7149   ins_cost(VOLATILE_REF_COST);
7150   format %{ "ldarb  $dst, $mem\t# byte" %}
7151 
7152   ins_encode(aarch64_enc_ldarb(dst, mem));
7153 
7154   ins_pipe(pipe_serial);
7155 %}
7156 
7157 // Load Byte (8 bit unsigned) into long
7158 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7159 %{
7160   match(Set dst (ConvI2L (LoadUB mem)));
7161 
7162   ins_cost(VOLATILE_REF_COST);
7163   format %{ "ldarb  $dst, $mem\t# byte" %}
7164 
7165   ins_encode(aarch64_enc_ldarb(dst, mem));
7166 
7167   ins_pipe(pipe_serial);
7168 %}
7169 
7170 // Load Short (16 bit signed)
7171 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7172 %{
7173   match(Set dst (LoadS mem));
7174 
7175   ins_cost(VOLATILE_REF_COST);
7176   format %{ "ldarshw  $dst, $mem\t# short" %}
7177 
7178   ins_encode(aarch64_enc_ldarshw(dst, mem));
7179 
7180   ins_pipe(pipe_serial);
7181 %}
7182 
7183 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7184 %{
7185   match(Set dst (LoadUS mem));
7186 
7187   ins_cost(VOLATILE_REF_COST);
7188   format %{ "ldarhw  $dst, $mem\t# short" %}
7189 
7190   ins_encode(aarch64_enc_ldarhw(dst, mem));
7191 
7192   ins_pipe(pipe_serial);
7193 %}
7194 
7195 // Load Short/Char (16 bit unsigned) into long
7196 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7197 %{
7198   match(Set dst (ConvI2L (LoadUS mem)));
7199 
7200   ins_cost(VOLATILE_REF_COST);
7201   format %{ "ldarh  $dst, $mem\t# short" %}
7202 
7203   ins_encode(aarch64_enc_ldarh(dst, mem));
7204 
7205   ins_pipe(pipe_serial);
7206 %}
7207 
7208 // Load Short/Char (16 bit signed) into long
7209 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7210 %{
7211   match(Set dst (ConvI2L (LoadS mem)));
7212 
7213   ins_cost(VOLATILE_REF_COST);
7214   format %{ "ldarh  $dst, $mem\t# short" %}
7215 
7216   ins_encode(aarch64_enc_ldarsh(dst, mem));
7217 
7218   ins_pipe(pipe_serial);
7219 %}
7220 
7221 // Load Integer (32 bit signed)
7222 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7223 %{
7224   match(Set dst (LoadI mem));
7225 
7226   ins_cost(VOLATILE_REF_COST);
7227   format %{ "ldarw  $dst, $mem\t# int" %}
7228 
7229   ins_encode(aarch64_enc_ldarw(dst, mem));
7230 
7231   ins_pipe(pipe_serial);
7232 %}
7233 
7234 // Load Integer (32 bit unsigned) into long
7235 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7236 %{
7237   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7238 
7239   ins_cost(VOLATILE_REF_COST);
7240   format %{ "ldarw  $dst, $mem\t# int" %}
7241 
7242   ins_encode(aarch64_enc_ldarw(dst, mem));
7243 
7244   ins_pipe(pipe_serial);
7245 %}
7246 
7247 // Load Long (64 bit signed)
7248 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7249 %{
7250   match(Set dst (LoadL mem));
7251 
7252   ins_cost(VOLATILE_REF_COST);
7253   format %{ "ldar  $dst, $mem\t# int" %}
7254 
7255   ins_encode(aarch64_enc_ldar(dst, mem));
7256 
7257   ins_pipe(pipe_serial);
7258 %}
7259 
7260 // Load Pointer
7261 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7262 %{
7263   match(Set dst (LoadP mem));
7264 
7265   ins_cost(VOLATILE_REF_COST);
7266   format %{ "ldar  $dst, $mem\t# ptr" %}
7267 
7268   ins_encode(aarch64_enc_ldar(dst, mem));
7269 
7270   ins_pipe(pipe_serial);
7271 %}
7272 
7273 // Load Compressed Pointer
7274 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7275 %{
7276   match(Set dst (LoadN mem));
7277 
7278   ins_cost(VOLATILE_REF_COST);
7279   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
7280 
7281   ins_encode(aarch64_enc_ldarw(dst, mem));
7282 
7283   ins_pipe(pipe_serial);
7284 %}
7285 
7286 // Load Float
7287 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7288 %{
7289   match(Set dst (LoadF mem));
7290 
7291   ins_cost(VOLATILE_REF_COST);
7292   format %{ "ldars  $dst, $mem\t# float" %}
7293 
7294   ins_encode( aarch64_enc_fldars(dst, mem) );
7295 
7296   ins_pipe(pipe_serial);
7297 %}
7298 
7299 // Load Double
7300 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7301 %{
7302   match(Set dst (LoadD mem));
7303 
7304   ins_cost(VOLATILE_REF_COST);
7305   format %{ "ldard  $dst, $mem\t# double" %}
7306 
7307   ins_encode( aarch64_enc_fldard(dst, mem) );
7308 
7309   ins_pipe(pipe_serial);
7310 %}
7311 
7312 // Store Byte
7313 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7314 %{
7315   match(Set mem (StoreB mem src));
7316 
7317   ins_cost(VOLATILE_REF_COST);
7318   format %{ "stlrb  $src, $mem\t# byte" %}
7319 
7320   ins_encode(aarch64_enc_stlrb(src, mem));
7321 
7322   ins_pipe(pipe_class_memory);
7323 %}
7324 
7325 // Store Char/Short
7326 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7327 %{
7328   match(Set mem (StoreC mem src));
7329 
7330   ins_cost(VOLATILE_REF_COST);
7331   format %{ "stlrh  $src, $mem\t# short" %}
7332 
7333   ins_encode(aarch64_enc_stlrh(src, mem));
7334 
7335   ins_pipe(pipe_class_memory);
7336 %}
7337 
7338 // Store Integer
7339 
7340 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7341 %{
7342   match(Set mem(StoreI mem src));
7343 
7344   ins_cost(VOLATILE_REF_COST);
7345   format %{ "stlrw  $src, $mem\t# int" %}
7346 
7347   ins_encode(aarch64_enc_stlrw(src, mem));
7348 
7349   ins_pipe(pipe_class_memory);
7350 %}
7351 
7352 // Store Long (64 bit signed)
7353 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7354 %{
7355   match(Set mem (StoreL mem src));
7356 
7357   ins_cost(VOLATILE_REF_COST);
7358   format %{ "stlr  $src, $mem\t# int" %}
7359 
7360   ins_encode(aarch64_enc_stlr(src, mem));
7361 
7362   ins_pipe(pipe_class_memory);
7363 %}
7364 
7365 // Store Pointer
7366 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7367 %{
7368   match(Set mem (StoreP mem src));
7369 
7370   ins_cost(VOLATILE_REF_COST);
7371   format %{ "stlr  $src, $mem\t# ptr" %}
7372 
7373   ins_encode(aarch64_enc_stlr(src, mem));
7374 
7375   ins_pipe(pipe_class_memory);
7376 %}
7377 
7378 // Store Compressed Pointer
7379 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7380 %{
7381   match(Set mem (StoreN mem src));
7382 
7383   ins_cost(VOLATILE_REF_COST);
7384   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
7385 
7386   ins_encode(aarch64_enc_stlrw(src, mem));
7387 
7388   ins_pipe(pipe_class_memory);
7389 %}
7390 
7391 // Store Float
7392 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7393 %{
7394   match(Set mem (StoreF mem src));
7395 
7396   ins_cost(VOLATILE_REF_COST);
7397   format %{ "stlrs  $src, $mem\t# float" %}
7398 
7399   ins_encode( aarch64_enc_fstlrs(src, mem) );
7400 
7401   ins_pipe(pipe_class_memory);
7402 %}
7403 
7404 // TODO
7405 // implement storeImmF0 and storeFImmPacked
7406 
7407 // Store Double
7408 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7409 %{
7410   match(Set mem (StoreD mem src));
7411 
7412   ins_cost(VOLATILE_REF_COST);
7413   format %{ "stlrd  $src, $mem\t# double" %}
7414 
7415   ins_encode( aarch64_enc_fstlrd(src, mem) );
7416 
7417   ins_pipe(pipe_class_memory);
7418 %}
7419 
7420 //  ---------------- end of volatile loads and stores ----------------
7421 
7422 // ============================================================================
7423 // BSWAP Instructions
7424 
7425 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7426   match(Set dst (ReverseBytesI src));
7427 
7428   ins_cost(INSN_COST);
7429   format %{ "revw  $dst, $src" %}
7430 
7431   ins_encode %{
7432     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7433   %}
7434 
7435   ins_pipe(ialu_reg);
7436 %}
7437 
7438 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7439   match(Set dst (ReverseBytesL src));
7440 
7441   ins_cost(INSN_COST);
7442   format %{ "rev  $dst, $src" %}
7443 
7444   ins_encode %{
7445     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7446   %}
7447 
7448   ins_pipe(ialu_reg);
7449 %}
7450 
7451 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7452   match(Set dst (ReverseBytesUS src));
7453 
7454   ins_cost(INSN_COST);
7455   format %{ "rev16w  $dst, $src" %}
7456 
7457   ins_encode %{
7458     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7459   %}
7460 
7461   ins_pipe(ialu_reg);
7462 %}
7463 
7464 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7465   match(Set dst (ReverseBytesS src));
7466 
7467   ins_cost(INSN_COST);
7468   format %{ "rev16w  $dst, $src\n\t"
7469             "sbfmw $dst, $dst, #0, #15" %}
7470 
7471   ins_encode %{
7472     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7473     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7474   %}
7475 
7476   ins_pipe(ialu_reg);
7477 %}
7478 
7479 // ============================================================================
7480 // Zero Count Instructions
7481 
7482 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7483   match(Set dst (CountLeadingZerosI src));
7484 
7485   ins_cost(INSN_COST);
7486   format %{ "clzw  $dst, $src" %}
7487   ins_encode %{
7488     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7489   %}
7490 
7491   ins_pipe(ialu_reg);
7492 %}
7493 
7494 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7495   match(Set dst (CountLeadingZerosL src));
7496 
7497   ins_cost(INSN_COST);
7498   format %{ "clz   $dst, $src" %}
7499   ins_encode %{
7500     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7501   %}
7502 
7503   ins_pipe(ialu_reg);
7504 %}
7505 
7506 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7507   match(Set dst (CountTrailingZerosI src));
7508 
7509   ins_cost(INSN_COST * 2);
7510   format %{ "rbitw  $dst, $src\n\t"
7511             "clzw   $dst, $dst" %}
7512   ins_encode %{
7513     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7514     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7515   %}
7516 
7517   ins_pipe(ialu_reg);
7518 %}
7519 
7520 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7521   match(Set dst (CountTrailingZerosL src));
7522 
7523   ins_cost(INSN_COST * 2);
7524   format %{ "rbit   $dst, $src\n\t"
7525             "clz    $dst, $dst" %}
7526   ins_encode %{
7527     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7528     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7529   %}
7530 
7531   ins_pipe(ialu_reg);
7532 %}
7533 
7534 //---------- Population Count Instructions -------------------------------------
7535 //
7536 
7537 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7538   predicate(UsePopCountInstruction);
7539   match(Set dst (PopCountI src));
7540   effect(TEMP tmp);
7541   ins_cost(INSN_COST * 13);
7542 
7543   format %{ "movw   $src, $src\n\t"
7544             "mov    $tmp, $src\t# vector (1D)\n\t"
7545             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7546             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7547             "mov    $dst, $tmp\t# vector (1D)" %}
7548   ins_encode %{
7549     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7550     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7551     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7552     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7553     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7554   %}
7555 
7556   ins_pipe(pipe_class_default);
7557 %}
7558 
7559 instruct popCountI_mem(iRegINoSp dst, memory mem, vRegF tmp) %{
7560   predicate(UsePopCountInstruction);
7561   match(Set dst (PopCountI (LoadI mem)));
7562   effect(TEMP tmp);
7563   ins_cost(INSN_COST * 13);
7564 
7565   format %{ "ldrs   $tmp, $mem\n\t"
7566             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7567             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7568             "mov    $dst, $tmp\t# vector (1D)" %}
7569   ins_encode %{
7570     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7571     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7572                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7573     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7574     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7575     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7576   %}
7577 
7578   ins_pipe(pipe_class_default);
7579 %}
7580 
7581 // Note: Long.bitCount(long) returns an int.
7582 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7583   predicate(UsePopCountInstruction);
7584   match(Set dst (PopCountL src));
7585   effect(TEMP tmp);
7586   ins_cost(INSN_COST * 13);
7587 
7588   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
7589             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7590             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7591             "mov    $dst, $tmp\t# vector (1D)" %}
7592   ins_encode %{
7593     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7594     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7595     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7596     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7597   %}
7598 
7599   ins_pipe(pipe_class_default);
7600 %}
7601 
7602 instruct popCountL_mem(iRegINoSp dst, memory mem, vRegD tmp) %{
7603   predicate(UsePopCountInstruction);
7604   match(Set dst (PopCountL (LoadL mem)));
7605   effect(TEMP tmp);
7606   ins_cost(INSN_COST * 13);
7607 
7608   format %{ "ldrd   $tmp, $mem\n\t"
7609             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
7610             "addv   $tmp, $tmp\t# vector (8B)\n\t"
7611             "mov    $dst, $tmp\t# vector (1D)" %}
7612   ins_encode %{
7613     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7614     loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7615                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7616     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7617     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7618     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7619   %}
7620 
7621   ins_pipe(pipe_class_default);
7622 %}
7623 
7624 // ============================================================================
7625 // MemBar Instruction
7626 
7627 instruct load_fence() %{
7628   match(LoadFence);
7629   ins_cost(VOLATILE_REF_COST);
7630 
7631   format %{ "load_fence" %}
7632 
7633   ins_encode %{
7634     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7635   %}
7636   ins_pipe(pipe_serial);
7637 %}
7638 
7639 instruct unnecessary_membar_acquire() %{
7640   predicate(unnecessary_acquire(n));
7641   match(MemBarAcquire);
7642   ins_cost(0);
7643 
7644   format %{ "membar_acquire (elided)" %}
7645 
7646   ins_encode %{
7647     __ block_comment("membar_acquire (elided)");
7648   %}
7649 
7650   ins_pipe(pipe_class_empty);
7651 %}
7652 
7653 instruct membar_acquire() %{
7654   match(MemBarAcquire);
7655   ins_cost(VOLATILE_REF_COST);
7656 
7657   format %{ "membar_acquire\n\t"
7658             "dmb ish" %}
7659 
7660   ins_encode %{
7661     __ block_comment("membar_acquire");
7662     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7663   %}
7664 
7665   ins_pipe(pipe_serial);
7666 %}
7667 
7668 
7669 instruct membar_acquire_lock() %{
7670   match(MemBarAcquireLock);
7671   ins_cost(VOLATILE_REF_COST);
7672 
7673   format %{ "membar_acquire_lock (elided)" %}
7674 
7675   ins_encode %{
7676     __ block_comment("membar_acquire_lock (elided)");
7677   %}
7678 
7679   ins_pipe(pipe_serial);
7680 %}
7681 
7682 instruct store_fence() %{
7683   match(StoreFence);
7684   ins_cost(VOLATILE_REF_COST);
7685 
7686   format %{ "store_fence" %}
7687 
7688   ins_encode %{
7689     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7690   %}
7691   ins_pipe(pipe_serial);
7692 %}
7693 
7694 instruct unnecessary_membar_release() %{
7695   predicate(unnecessary_release(n));
7696   match(MemBarRelease);
7697   ins_cost(0);
7698 
7699   format %{ "membar_release (elided)" %}
7700 
7701   ins_encode %{
7702     __ block_comment("membar_release (elided)");
7703   %}
7704   ins_pipe(pipe_serial);
7705 %}
7706 
7707 instruct membar_release() %{
7708   match(MemBarRelease);
7709   ins_cost(VOLATILE_REF_COST);
7710 
7711   format %{ "membar_release\n\t"
7712             "dmb ish" %}
7713 
7714   ins_encode %{
7715     __ block_comment("membar_release");
7716     __ membar(Assembler::LoadStore|Assembler::StoreStore);
7717   %}
7718   ins_pipe(pipe_serial);
7719 %}
7720 
7721 instruct membar_storestore() %{
7722   match(MemBarStoreStore);
7723   ins_cost(VOLATILE_REF_COST);
7724 
7725   format %{ "MEMBAR-store-store" %}
7726 
7727   ins_encode %{
7728     __ membar(Assembler::StoreStore);
7729   %}
7730   ins_pipe(pipe_serial);
7731 %}
7732 
7733 instruct membar_release_lock() %{
7734   match(MemBarReleaseLock);
7735   ins_cost(VOLATILE_REF_COST);
7736 
7737   format %{ "membar_release_lock (elided)" %}
7738 
7739   ins_encode %{
7740     __ block_comment("membar_release_lock (elided)");
7741   %}
7742 
7743   ins_pipe(pipe_serial);
7744 %}
7745 
7746 instruct unnecessary_membar_volatile() %{
7747   predicate(unnecessary_volatile(n));
7748   match(MemBarVolatile);
7749   ins_cost(0);
7750 
7751   format %{ "membar_volatile (elided)" %}
7752 
7753   ins_encode %{
7754     __ block_comment("membar_volatile (elided)");
7755   %}
7756 
7757   ins_pipe(pipe_serial);
7758 %}
7759 
7760 instruct membar_volatile() %{
7761   match(MemBarVolatile);
7762   ins_cost(VOLATILE_REF_COST*100);
7763 
7764   format %{ "membar_volatile\n\t"
7765              "dmb ish"%}
7766 
7767   ins_encode %{
7768     __ block_comment("membar_volatile");
7769     __ membar(Assembler::StoreLoad);
7770   %}
7771 
7772   ins_pipe(pipe_serial);
7773 %}
7774 
7775 // ============================================================================
7776 // Cast/Convert Instructions
7777 
7778 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7779   match(Set dst (CastX2P src));
7780 
7781   ins_cost(INSN_COST);
7782   format %{ "mov $dst, $src\t# long -> ptr" %}
7783 
7784   ins_encode %{
7785     if ($dst$$reg != $src$$reg) {
7786       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7787     }
7788   %}
7789 
7790   ins_pipe(ialu_reg);
7791 %}
7792 
7793 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7794   match(Set dst (CastP2X src));
7795 
7796   ins_cost(INSN_COST);
7797   format %{ "mov $dst, $src\t# ptr -> long" %}
7798 
7799   ins_encode %{
7800     if ($dst$$reg != $src$$reg) {
7801       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7802     }
7803   %}
7804 
7805   ins_pipe(ialu_reg);
7806 %}
7807 
7808 // Convert oop into int for vectors alignment masking
7809 instruct convP2I(iRegINoSp dst, iRegP src) %{
7810   match(Set dst (ConvL2I (CastP2X src)));
7811 
7812   ins_cost(INSN_COST);
7813   format %{ "movw $dst, $src\t# ptr -> int" %}
7814   ins_encode %{
7815     __ movw($dst$$Register, $src$$Register);
7816   %}
7817 
7818   ins_pipe(ialu_reg);
7819 %}
7820 
7821 // Convert compressed oop into int for vectors alignment masking
7822 // in case of 32bit oops (heap < 4Gb).
7823 instruct convN2I(iRegINoSp dst, iRegN src)
7824 %{
7825   predicate(CompressedOops::shift() == 0);
7826   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7827 
7828   ins_cost(INSN_COST);
7829   format %{ "mov dst, $src\t# compressed ptr -> int" %}
7830   ins_encode %{
7831     __ movw($dst$$Register, $src$$Register);
7832   %}
7833 
7834   ins_pipe(ialu_reg);
7835 %}
7836 
7837 
7838 // Convert oop pointer into compressed form
7839 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7840   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7841   match(Set dst (EncodeP src));
7842   effect(KILL cr);
7843   ins_cost(INSN_COST * 3);
7844   format %{ "encode_heap_oop $dst, $src" %}
7845   ins_encode %{
7846     Register s = $src$$Register;
7847     Register d = $dst$$Register;
7848     __ encode_heap_oop(d, s);
7849   %}
7850   ins_pipe(ialu_reg);
7851 %}
7852 
7853 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7854   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7855   match(Set dst (EncodeP src));
7856   ins_cost(INSN_COST * 3);
7857   format %{ "encode_heap_oop_not_null $dst, $src" %}
7858   ins_encode %{
7859     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7860   %}
7861   ins_pipe(ialu_reg);
7862 %}
7863 
7864 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7865   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
7866             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
7867   match(Set dst (DecodeN src));
7868   ins_cost(INSN_COST * 3);
7869   format %{ "decode_heap_oop $dst, $src" %}
7870   ins_encode %{
7871     Register s = $src$$Register;
7872     Register d = $dst$$Register;
7873     __ decode_heap_oop(d, s);
7874   %}
7875   ins_pipe(ialu_reg);
7876 %}
7877 
7878 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7879   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7880             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7881   match(Set dst (DecodeN src));
7882   ins_cost(INSN_COST * 3);
7883   format %{ "decode_heap_oop_not_null $dst, $src" %}
7884   ins_encode %{
7885     Register s = $src$$Register;
7886     Register d = $dst$$Register;
7887     __ decode_heap_oop_not_null(d, s);
7888   %}
7889   ins_pipe(ialu_reg);
7890 %}
7891 
7892 // n.b. AArch64 implementations of encode_klass_not_null and
7893 // decode_klass_not_null do not modify the flags register so, unlike
7894 // Intel, we don't kill CR as a side effect here
7895 
7896 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
7897   match(Set dst (EncodePKlass src));
7898 
7899   ins_cost(INSN_COST * 3);
7900   format %{ "encode_klass_not_null $dst,$src" %}
7901 
7902   ins_encode %{
7903     Register src_reg = as_Register($src$$reg);
7904     Register dst_reg = as_Register($dst$$reg);
7905     __ encode_klass_not_null(dst_reg, src_reg);
7906   %}
7907 
7908    ins_pipe(ialu_reg);
7909 %}
7910 
7911 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
7912   match(Set dst (DecodeNKlass src));
7913 
7914   ins_cost(INSN_COST * 3);
7915   format %{ "decode_klass_not_null $dst,$src" %}
7916 
7917   ins_encode %{
7918     Register src_reg = as_Register($src$$reg);
7919     Register dst_reg = as_Register($dst$$reg);
7920     if (dst_reg != src_reg) {
7921       __ decode_klass_not_null(dst_reg, src_reg);
7922     } else {
7923       __ decode_klass_not_null(dst_reg);
7924     }
7925   %}
7926 
7927    ins_pipe(ialu_reg);
7928 %}
7929 
7930 instruct checkCastPP(iRegPNoSp dst)
7931 %{
7932   match(Set dst (CheckCastPP dst));
7933 
7934   size(0);
7935   format %{ "# checkcastPP of $dst" %}
7936   ins_encode(/* empty encoding */);
7937   ins_pipe(pipe_class_empty);
7938 %}
7939 
7940 instruct castPP(iRegPNoSp dst)
7941 %{
7942   match(Set dst (CastPP dst));
7943 
7944   size(0);
7945   format %{ "# castPP of $dst" %}
7946   ins_encode(/* empty encoding */);
7947   ins_pipe(pipe_class_empty);
7948 %}
7949 
7950 instruct castII(iRegI dst)
7951 %{
7952   match(Set dst (CastII dst));
7953 
7954   size(0);
7955   format %{ "# castII of $dst" %}
7956   ins_encode(/* empty encoding */);
7957   ins_cost(0);
7958   ins_pipe(pipe_class_empty);
7959 %}
7960 
7961 // ============================================================================
7962 // Atomic operation instructions
7963 //
7964 // Intel and SPARC both implement Ideal Node LoadPLocked and
7965 // Store{PIL}Conditional instructions using a normal load for the
7966 // LoadPLocked and a CAS for the Store{PIL}Conditional.
7967 //
7968 // The ideal code appears only to use LoadPLocked/StorePLocked as a
7969 // pair to lock object allocations from Eden space when not using
7970 // TLABs.
7971 //
7972 // There does not appear to be a Load{IL}Locked Ideal Node and the
7973 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
7974 // and to use StoreIConditional only for 32-bit and StoreLConditional
7975 // only for 64-bit.
7976 //
7977 // We implement LoadPLocked and StorePLocked instructions using,
7978 // respectively the AArch64 hw load-exclusive and store-conditional
7979 // instructions. Whereas we must implement each of
7980 // Store{IL}Conditional using a CAS which employs a pair of
7981 // instructions comprising a load-exclusive followed by a
7982 // store-conditional.
7983 
7984 
7985 // Locked-load (linked load) of the current heap-top
7986 // used when updating the eden heap top
7987 // implemented using ldaxr on AArch64
7988 
7989 instruct loadPLocked(iRegPNoSp dst, indirect mem)
7990 %{
7991   match(Set dst (LoadPLocked mem));
7992 
7993   ins_cost(VOLATILE_REF_COST);
7994 
7995   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
7996 
7997   ins_encode(aarch64_enc_ldaxr(dst, mem));
7998 
7999   ins_pipe(pipe_serial);
8000 %}
8001 
8002 // Conditional-store of the updated heap-top.
8003 // Used during allocation of the shared heap.
8004 // Sets flag (EQ) on success.
8005 // implemented using stlxr on AArch64.
8006 
8007 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8008 %{
8009   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8010 
8011   ins_cost(VOLATILE_REF_COST);
8012 
8013  // TODO
8014  // do we need to do a store-conditional release or can we just use a
8015  // plain store-conditional?
8016 
8017   format %{
8018     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8019     "cmpw rscratch1, zr\t# EQ on successful write"
8020   %}
8021 
8022   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8023 
8024   ins_pipe(pipe_serial);
8025 %}
8026 
8027 
8028 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
8029 // when attempting to rebias a lock towards the current thread.  We
8030 // must use the acquire form of cmpxchg in order to guarantee acquire
8031 // semantics in this case.
8032 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8033 %{
8034   match(Set cr (StoreLConditional mem (Binary oldval newval)));
8035 
8036   ins_cost(VOLATILE_REF_COST);
8037 
8038   format %{
8039     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8040     "cmpw rscratch1, zr\t# EQ on successful write"
8041   %}
8042 
8043   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
8044 
8045   ins_pipe(pipe_slow);
8046 %}
8047 
8048 // storeIConditional also has acquire semantics, for no better reason
8049 // than matching storeLConditional.  At the time of writing this
8050 // comment storeIConditional was not used anywhere by AArch64.
8051 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8052 %{
8053   match(Set cr (StoreIConditional mem (Binary oldval newval)));
8054 
8055   ins_cost(VOLATILE_REF_COST);
8056 
8057   format %{
8058     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8059     "cmpw rscratch1, zr\t# EQ on successful write"
8060   %}
8061 
8062   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8063 
8064   ins_pipe(pipe_slow);
8065 %}
8066 
8067 // standard CompareAndSwapX when we are using barriers
8068 // these have higher priority than the rules selected by a predicate
8069 
8070 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8071 // can't match them
8072 
8073 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8074 
8075   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8076   ins_cost(2 * VOLATILE_REF_COST);
8077 
8078   effect(KILL cr);
8079 
8080   format %{
8081     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8082     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8083   %}
8084 
8085   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8086             aarch64_enc_cset_eq(res));
8087 
8088   ins_pipe(pipe_slow);
8089 %}
8090 
8091 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8092 
8093   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8094   ins_cost(2 * VOLATILE_REF_COST);
8095 
8096   effect(KILL cr);
8097 
8098   format %{
8099     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8100     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8101   %}
8102 
8103   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8104             aarch64_enc_cset_eq(res));
8105 
8106   ins_pipe(pipe_slow);
8107 %}
8108 
8109 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8110 
8111   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8112   ins_cost(2 * VOLATILE_REF_COST);
8113 
8114   effect(KILL cr);
8115 
8116  format %{
8117     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8118     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8119  %}
8120 
8121  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8122             aarch64_enc_cset_eq(res));
8123 
8124   ins_pipe(pipe_slow);
8125 %}
8126 
8127 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8128 
8129   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8130   ins_cost(2 * VOLATILE_REF_COST);
8131 
8132   effect(KILL cr);
8133 
8134  format %{
8135     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8136     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8137  %}
8138 
8139  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8140             aarch64_enc_cset_eq(res));
8141 
8142   ins_pipe(pipe_slow);
8143 %}
8144 
8145 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8146 
8147   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8148   ins_cost(2 * VOLATILE_REF_COST);
8149 
8150   effect(KILL cr);
8151 
8152  format %{
8153     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8154     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8155  %}
8156 
8157  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8158             aarch64_enc_cset_eq(res));
8159 
8160   ins_pipe(pipe_slow);
8161 %}
8162 
8163 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8164 
8165   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8166   ins_cost(2 * VOLATILE_REF_COST);
8167 
8168   effect(KILL cr);
8169 
8170  format %{
8171     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8172     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8173  %}
8174 
8175  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8176             aarch64_enc_cset_eq(res));
8177 
8178   ins_pipe(pipe_slow);
8179 %}
8180 
8181 // alternative CompareAndSwapX when we are eliding barriers
8182 
8183 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8184 
8185   predicate(needs_acquiring_load_exclusive(n));
8186   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8187   ins_cost(VOLATILE_REF_COST);
8188 
8189   effect(KILL cr);
8190 
8191   format %{
8192     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8193     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8194   %}
8195 
8196   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8197             aarch64_enc_cset_eq(res));
8198 
8199   ins_pipe(pipe_slow);
8200 %}
8201 
8202 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8203 
8204   predicate(needs_acquiring_load_exclusive(n));
8205   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8206   ins_cost(VOLATILE_REF_COST);
8207 
8208   effect(KILL cr);
8209 
8210   format %{
8211     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8212     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8213   %}
8214 
8215   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8216             aarch64_enc_cset_eq(res));
8217 
8218   ins_pipe(pipe_slow);
8219 %}
8220 
8221 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8222 
8223   predicate(needs_acquiring_load_exclusive(n));
8224   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8225   ins_cost(VOLATILE_REF_COST);
8226 
8227   effect(KILL cr);
8228 
8229  format %{
8230     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8231     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8232  %}
8233 
8234  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8235             aarch64_enc_cset_eq(res));
8236 
8237   ins_pipe(pipe_slow);
8238 %}
8239 
8240 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8241 
8242   predicate(needs_acquiring_load_exclusive(n));
8243   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8244   ins_cost(VOLATILE_REF_COST);
8245 
8246   effect(KILL cr);
8247 
8248  format %{
8249     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8250     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8251  %}
8252 
8253  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8254             aarch64_enc_cset_eq(res));
8255 
8256   ins_pipe(pipe_slow);
8257 %}
8258 
8259 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8260 
8261   predicate(needs_acquiring_load_exclusive(n));
8262   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8263   ins_cost(VOLATILE_REF_COST);
8264 
8265   effect(KILL cr);
8266 
8267  format %{
8268     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8269     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8270  %}
8271 
8272  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8273             aarch64_enc_cset_eq(res));
8274 
8275   ins_pipe(pipe_slow);
8276 %}
8277 
8278 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8279 
8280   predicate(needs_acquiring_load_exclusive(n));
8281   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8282   ins_cost(VOLATILE_REF_COST);
8283 
8284   effect(KILL cr);
8285 
8286  format %{
8287     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8288     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8289  %}
8290 
8291  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8292             aarch64_enc_cset_eq(res));
8293 
8294   ins_pipe(pipe_slow);
8295 %}
8296 
8297 
8298 // ---------------------------------------------------------------------
8299 
8300 
8301 // BEGIN This section of the file is automatically generated. Do not edit --------------
8302 
8303 // Sundry CAS operations.  Note that release is always true,
8304 // regardless of the memory ordering of the CAS.  This is because we
8305 // need the volatile case to be sequentially consistent but there is
8306 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
8307 // can't check the type of memory ordering here, so we always emit a
8308 // STLXR.
8309 
8310 // This section is generated from aarch64_ad_cas.m4
8311 
8312 
8313 
8314 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8315   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8316   ins_cost(2 * VOLATILE_REF_COST);
8317   effect(TEMP_DEF res, KILL cr);
8318   format %{
8319     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8320   %}
8321   ins_encode %{
8322     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8323                Assembler::byte, /*acquire*/ false, /*release*/ true,
8324                /*weak*/ false, $res$$Register);
8325     __ sxtbw($res$$Register, $res$$Register);
8326   %}
8327   ins_pipe(pipe_slow);
8328 %}
8329 
8330 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8331   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8332   ins_cost(2 * VOLATILE_REF_COST);
8333   effect(TEMP_DEF res, KILL cr);
8334   format %{
8335     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8336   %}
8337   ins_encode %{
8338     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8339                Assembler::halfword, /*acquire*/ false, /*release*/ true,
8340                /*weak*/ false, $res$$Register);
8341     __ sxthw($res$$Register, $res$$Register);
8342   %}
8343   ins_pipe(pipe_slow);
8344 %}
8345 
8346 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8347   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8348   ins_cost(2 * VOLATILE_REF_COST);
8349   effect(TEMP_DEF res, KILL cr);
8350   format %{
8351     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8352   %}
8353   ins_encode %{
8354     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8355                Assembler::word, /*acquire*/ false, /*release*/ true,
8356                /*weak*/ false, $res$$Register);
8357   %}
8358   ins_pipe(pipe_slow);
8359 %}
8360 
8361 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8362   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8363   ins_cost(2 * VOLATILE_REF_COST);
8364   effect(TEMP_DEF res, KILL cr);
8365   format %{
8366     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8367   %}
8368   ins_encode %{
8369     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8370                Assembler::xword, /*acquire*/ false, /*release*/ true,
8371                /*weak*/ false, $res$$Register);
8372   %}
8373   ins_pipe(pipe_slow);
8374 %}
8375 
8376 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8377   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8378   ins_cost(2 * VOLATILE_REF_COST);
8379   effect(TEMP_DEF res, KILL cr);
8380   format %{
8381     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8382   %}
8383   ins_encode %{
8384     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8385                Assembler::word, /*acquire*/ false, /*release*/ true,
8386                /*weak*/ false, $res$$Register);
8387   %}
8388   ins_pipe(pipe_slow);
8389 %}
8390 
8391 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8392   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8393   ins_cost(2 * VOLATILE_REF_COST);
8394   effect(TEMP_DEF res, KILL cr);
8395   format %{
8396     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8397   %}
8398   ins_encode %{
8399     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8400                Assembler::xword, /*acquire*/ false, /*release*/ true,
8401                /*weak*/ false, $res$$Register);
8402   %}
8403   ins_pipe(pipe_slow);
8404 %}
8405 
8406 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8407   predicate(needs_acquiring_load_exclusive(n));
8408   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8409   ins_cost(VOLATILE_REF_COST);
8410   effect(TEMP_DEF res, KILL cr);
8411   format %{
8412     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8413   %}
8414   ins_encode %{
8415     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8416                Assembler::byte, /*acquire*/ true, /*release*/ true,
8417                /*weak*/ false, $res$$Register);
8418     __ sxtbw($res$$Register, $res$$Register);
8419   %}
8420   ins_pipe(pipe_slow);
8421 %}
8422 
8423 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8424   predicate(needs_acquiring_load_exclusive(n));
8425   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8426   ins_cost(VOLATILE_REF_COST);
8427   effect(TEMP_DEF res, KILL cr);
8428   format %{
8429     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8430   %}
8431   ins_encode %{
8432     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8433                Assembler::halfword, /*acquire*/ true, /*release*/ true,
8434                /*weak*/ false, $res$$Register);
8435     __ sxthw($res$$Register, $res$$Register);
8436   %}
8437   ins_pipe(pipe_slow);
8438 %}
8439 
8440 
8441 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8442   predicate(needs_acquiring_load_exclusive(n));
8443   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8444   ins_cost(VOLATILE_REF_COST);
8445   effect(TEMP_DEF res, KILL cr);
8446   format %{
8447     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8448   %}
8449   ins_encode %{
8450     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8451                Assembler::word, /*acquire*/ true, /*release*/ true,
8452                /*weak*/ false, $res$$Register);
8453   %}
8454   ins_pipe(pipe_slow);
8455 %}
8456 
8457 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8458   predicate(needs_acquiring_load_exclusive(n));
8459   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8460   ins_cost(VOLATILE_REF_COST);
8461   effect(TEMP_DEF res, KILL cr);
8462   format %{
8463     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8464   %}
8465   ins_encode %{
8466     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8467                Assembler::xword, /*acquire*/ true, /*release*/ true,
8468                /*weak*/ false, $res$$Register);
8469   %}
8470   ins_pipe(pipe_slow);
8471 %}
8472 
8473 
8474 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8475   predicate(needs_acquiring_load_exclusive(n));
8476   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8477   ins_cost(VOLATILE_REF_COST);
8478   effect(TEMP_DEF res, KILL cr);
8479   format %{
8480     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8481   %}
8482   ins_encode %{
8483     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8484                Assembler::word, /*acquire*/ true, /*release*/ true,
8485                /*weak*/ false, $res$$Register);
8486   %}
8487   ins_pipe(pipe_slow);
8488 %}
8489 
8490 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8491   predicate(needs_acquiring_load_exclusive(n));
8492   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8493   ins_cost(VOLATILE_REF_COST);
8494   effect(TEMP_DEF res, KILL cr);
8495   format %{
8496     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8497   %}
8498   ins_encode %{
8499     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8500                Assembler::xword, /*acquire*/ true, /*release*/ true,
8501                /*weak*/ false, $res$$Register);
8502   %}
8503   ins_pipe(pipe_slow);
8504 %}
8505 
8506 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8507   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8508   ins_cost(2 * VOLATILE_REF_COST);
8509   effect(KILL cr);
8510   format %{
8511     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8512     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8513   %}
8514   ins_encode %{
8515     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8516                Assembler::byte, /*acquire*/ false, /*release*/ true,
8517                /*weak*/ true, noreg);
8518     __ csetw($res$$Register, Assembler::EQ);
8519   %}
8520   ins_pipe(pipe_slow);
8521 %}
8522 
8523 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8524   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8525   ins_cost(2 * VOLATILE_REF_COST);
8526   effect(KILL cr);
8527   format %{
8528     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8529     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8530   %}
8531   ins_encode %{
8532     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8533                Assembler::halfword, /*acquire*/ false, /*release*/ true,
8534                /*weak*/ true, noreg);
8535     __ csetw($res$$Register, Assembler::EQ);
8536   %}
8537   ins_pipe(pipe_slow);
8538 %}
8539 
8540 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8541   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8542   ins_cost(2 * VOLATILE_REF_COST);
8543   effect(KILL cr);
8544   format %{
8545     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8546     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8547   %}
8548   ins_encode %{
8549     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8550                Assembler::word, /*acquire*/ false, /*release*/ true,
8551                /*weak*/ true, noreg);
8552     __ csetw($res$$Register, Assembler::EQ);
8553   %}
8554   ins_pipe(pipe_slow);
8555 %}
8556 
8557 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8558   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8559   ins_cost(2 * VOLATILE_REF_COST);
8560   effect(KILL cr);
8561   format %{
8562     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8563     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8564   %}
8565   ins_encode %{
8566     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8567                Assembler::xword, /*acquire*/ false, /*release*/ true,
8568                /*weak*/ true, noreg);
8569     __ csetw($res$$Register, Assembler::EQ);
8570   %}
8571   ins_pipe(pipe_slow);
8572 %}
8573 
8574 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8575   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8576   ins_cost(2 * VOLATILE_REF_COST);
8577   effect(KILL cr);
8578   format %{
8579     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8580     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8581   %}
8582   ins_encode %{
8583     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8584                Assembler::word, /*acquire*/ false, /*release*/ true,
8585                /*weak*/ true, noreg);
8586     __ csetw($res$$Register, Assembler::EQ);
8587   %}
8588   ins_pipe(pipe_slow);
8589 %}
8590 
8591 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8592   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8593   ins_cost(2 * VOLATILE_REF_COST);
8594   effect(KILL cr);
8595   format %{
8596     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8597     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8598   %}
8599   ins_encode %{
8600     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8601                Assembler::xword, /*acquire*/ false, /*release*/ true,
8602                /*weak*/ true, noreg);
8603     __ csetw($res$$Register, Assembler::EQ);
8604   %}
8605   ins_pipe(pipe_slow);
8606 %}
8607 
8608 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8609   predicate(needs_acquiring_load_exclusive(n));
8610   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8611   ins_cost(VOLATILE_REF_COST);
8612   effect(KILL cr);
8613   format %{
8614     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8615     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8616   %}
8617   ins_encode %{
8618     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8619                Assembler::byte, /*acquire*/ true, /*release*/ true,
8620                /*weak*/ true, noreg);
8621     __ csetw($res$$Register, Assembler::EQ);
8622   %}
8623   ins_pipe(pipe_slow);
8624 %}
8625 
8626 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8627   predicate(needs_acquiring_load_exclusive(n));
8628   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8629   ins_cost(VOLATILE_REF_COST);
8630   effect(KILL cr);
8631   format %{
8632     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8633     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8634   %}
8635   ins_encode %{
8636     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8637                Assembler::halfword, /*acquire*/ true, /*release*/ true,
8638                /*weak*/ true, noreg);
8639     __ csetw($res$$Register, Assembler::EQ);
8640   %}
8641   ins_pipe(pipe_slow);
8642 %}
8643 
8644 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8645   predicate(needs_acquiring_load_exclusive(n));
8646   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8647   ins_cost(VOLATILE_REF_COST);
8648   effect(KILL cr);
8649   format %{
8650     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8651     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8652   %}
8653   ins_encode %{
8654     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8655                Assembler::word, /*acquire*/ true, /*release*/ true,
8656                /*weak*/ true, noreg);
8657     __ csetw($res$$Register, Assembler::EQ);
8658   %}
8659   ins_pipe(pipe_slow);
8660 %}
8661 
8662 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8663   predicate(needs_acquiring_load_exclusive(n));
8664   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8665   ins_cost(VOLATILE_REF_COST);
8666   effect(KILL cr);
8667   format %{
8668     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8669     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8670   %}
8671   ins_encode %{
8672     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8673                Assembler::xword, /*acquire*/ true, /*release*/ true,
8674                /*weak*/ true, noreg);
8675     __ csetw($res$$Register, Assembler::EQ);
8676   %}
8677   ins_pipe(pipe_slow);
8678 %}
8679 
8680 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8681   predicate(needs_acquiring_load_exclusive(n));
8682   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8683   ins_cost(VOLATILE_REF_COST);
8684   effect(KILL cr);
8685   format %{
8686     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8687     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8688   %}
8689   ins_encode %{
8690     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8691                Assembler::word, /*acquire*/ true, /*release*/ true,
8692                /*weak*/ true, noreg);
8693     __ csetw($res$$Register, Assembler::EQ);
8694   %}
8695   ins_pipe(pipe_slow);
8696 %}
8697 
8698 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8699   predicate(needs_acquiring_load_exclusive(n));
8700   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8701   ins_cost(VOLATILE_REF_COST);
8702   effect(KILL cr);
8703   format %{
8704     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8705     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8706   %}
8707   ins_encode %{
8708     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8709                Assembler::xword, /*acquire*/ true, /*release*/ true,
8710                /*weak*/ true, noreg);
8711     __ csetw($res$$Register, Assembler::EQ);
8712   %}
8713   ins_pipe(pipe_slow);
8714 %}
8715 
8716 // END This section of the file is automatically generated. Do not edit --------------
8717 // ---------------------------------------------------------------------
8718 
8719 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8720   match(Set prev (GetAndSetI mem newv));
8721   ins_cost(2 * VOLATILE_REF_COST);
8722   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
8723   ins_encode %{
8724     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8725   %}
8726   ins_pipe(pipe_serial);
8727 %}
8728 
8729 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8730   match(Set prev (GetAndSetL mem newv));
8731   ins_cost(2 * VOLATILE_REF_COST);
8732   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
8733   ins_encode %{
8734     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8735   %}
8736   ins_pipe(pipe_serial);
8737 %}
8738 
8739 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8740   match(Set prev (GetAndSetN mem newv));
8741   ins_cost(2 * VOLATILE_REF_COST);
8742   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8743   ins_encode %{
8744     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8745   %}
8746   ins_pipe(pipe_serial);
8747 %}
8748 
8749 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8750   match(Set prev (GetAndSetP mem newv));
8751   ins_cost(2 * VOLATILE_REF_COST);
8752   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
8753   ins_encode %{
8754     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8755   %}
8756   ins_pipe(pipe_serial);
8757 %}
8758 
8759 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8760   predicate(needs_acquiring_load_exclusive(n));
8761   match(Set prev (GetAndSetI mem newv));
8762   ins_cost(VOLATILE_REF_COST);
8763   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
8764   ins_encode %{
8765     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8766   %}
8767   ins_pipe(pipe_serial);
8768 %}
8769 
8770 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8771   predicate(needs_acquiring_load_exclusive(n));
8772   match(Set prev (GetAndSetL mem newv));
8773   ins_cost(VOLATILE_REF_COST);
8774   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
8775   ins_encode %{
8776     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8777   %}
8778   ins_pipe(pipe_serial);
8779 %}
8780 
8781 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8782   predicate(needs_acquiring_load_exclusive(n));
8783   match(Set prev (GetAndSetN mem newv));
8784   ins_cost(VOLATILE_REF_COST);
8785   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8786   ins_encode %{
8787     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8788   %}
8789   ins_pipe(pipe_serial);
8790 %}
8791 
8792 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8793   predicate(needs_acquiring_load_exclusive(n));
8794   match(Set prev (GetAndSetP mem newv));
8795   ins_cost(VOLATILE_REF_COST);
8796   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
8797   ins_encode %{
8798     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8799   %}
8800   ins_pipe(pipe_serial);
8801 %}
8802 
8803 
8804 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8805   match(Set newval (GetAndAddL mem incr));
8806   ins_cost(2 * VOLATILE_REF_COST + 1);
8807   format %{ "get_and_addL $newval, [$mem], $incr" %}
8808   ins_encode %{
8809     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8810   %}
8811   ins_pipe(pipe_serial);
8812 %}
8813 
8814 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8815   predicate(n->as_LoadStore()->result_not_used());
8816   match(Set dummy (GetAndAddL mem incr));
8817   ins_cost(2 * VOLATILE_REF_COST);
8818   format %{ "get_and_addL [$mem], $incr" %}
8819   ins_encode %{
8820     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
8821   %}
8822   ins_pipe(pipe_serial);
8823 %}
8824 
8825 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8826   match(Set newval (GetAndAddL mem incr));
8827   ins_cost(2 * VOLATILE_REF_COST + 1);
8828   format %{ "get_and_addL $newval, [$mem], $incr" %}
8829   ins_encode %{
8830     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
8831   %}
8832   ins_pipe(pipe_serial);
8833 %}
8834 
8835 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
8836   predicate(n->as_LoadStore()->result_not_used());
8837   match(Set dummy (GetAndAddL mem incr));
8838   ins_cost(2 * VOLATILE_REF_COST);
8839   format %{ "get_and_addL [$mem], $incr" %}
8840   ins_encode %{
8841     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
8842   %}
8843   ins_pipe(pipe_serial);
8844 %}
8845 
8846 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8847   match(Set newval (GetAndAddI mem incr));
8848   ins_cost(2 * VOLATILE_REF_COST + 1);
8849   format %{ "get_and_addI $newval, [$mem], $incr" %}
8850   ins_encode %{
8851     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8852   %}
8853   ins_pipe(pipe_serial);
8854 %}
8855 
8856 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
8857   predicate(n->as_LoadStore()->result_not_used());
8858   match(Set dummy (GetAndAddI mem incr));
8859   ins_cost(2 * VOLATILE_REF_COST);
8860   format %{ "get_and_addI [$mem], $incr" %}
8861   ins_encode %{
8862     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
8863   %}
8864   ins_pipe(pipe_serial);
8865 %}
8866 
8867 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8868   match(Set newval (GetAndAddI mem incr));
8869   ins_cost(2 * VOLATILE_REF_COST + 1);
8870   format %{ "get_and_addI $newval, [$mem], $incr" %}
8871   ins_encode %{
8872     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8873   %}
8874   ins_pipe(pipe_serial);
8875 %}
8876 
8877 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
8878   predicate(n->as_LoadStore()->result_not_used());
8879   match(Set dummy (GetAndAddI mem incr));
8880   ins_cost(2 * VOLATILE_REF_COST);
8881   format %{ "get_and_addI [$mem], $incr" %}
8882   ins_encode %{
8883     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
8884   %}
8885   ins_pipe(pipe_serial);
8886 %}
8887 
8888 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
8889   predicate(needs_acquiring_load_exclusive(n));
8890   match(Set newval (GetAndAddL mem incr));
8891   ins_cost(VOLATILE_REF_COST + 1);
8892   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8893   ins_encode %{
8894     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
8895   %}
8896   ins_pipe(pipe_serial);
8897 %}
8898 
8899 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
8900   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8901   match(Set dummy (GetAndAddL mem incr));
8902   ins_cost(VOLATILE_REF_COST);
8903   format %{ "get_and_addL_acq [$mem], $incr" %}
8904   ins_encode %{
8905     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
8906   %}
8907   ins_pipe(pipe_serial);
8908 %}
8909 
8910 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8911   predicate(needs_acquiring_load_exclusive(n));
8912   match(Set newval (GetAndAddL mem incr));
8913   ins_cost(VOLATILE_REF_COST + 1);
8914   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8915   ins_encode %{
8916     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
8917   %}
8918   ins_pipe(pipe_serial);
8919 %}
8920 
8921 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
8922   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8923   match(Set dummy (GetAndAddL mem incr));
8924   ins_cost(VOLATILE_REF_COST);
8925   format %{ "get_and_addL_acq [$mem], $incr" %}
8926   ins_encode %{
8927     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
8928   %}
8929   ins_pipe(pipe_serial);
8930 %}
8931 
8932 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8933   predicate(needs_acquiring_load_exclusive(n));
8934   match(Set newval (GetAndAddI mem incr));
8935   ins_cost(VOLATILE_REF_COST + 1);
8936   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8937   ins_encode %{
8938     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8939   %}
8940   ins_pipe(pipe_serial);
8941 %}
8942 
8943 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
8944   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8945   match(Set dummy (GetAndAddI mem incr));
8946   ins_cost(VOLATILE_REF_COST);
8947   format %{ "get_and_addI_acq [$mem], $incr" %}
8948   ins_encode %{
8949     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
8950   %}
8951   ins_pipe(pipe_serial);
8952 %}
8953 
8954 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8955   predicate(needs_acquiring_load_exclusive(n));
8956   match(Set newval (GetAndAddI mem incr));
8957   ins_cost(VOLATILE_REF_COST + 1);
8958   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8959   ins_encode %{
8960     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8961   %}
8962   ins_pipe(pipe_serial);
8963 %}
8964 
8965 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
8966   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8967   match(Set dummy (GetAndAddI mem incr));
8968   ins_cost(VOLATILE_REF_COST);
8969   format %{ "get_and_addI_acq [$mem], $incr" %}
8970   ins_encode %{
8971     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
8972   %}
8973   ins_pipe(pipe_serial);
8974 %}
8975 
8976 // Manifest a CmpL result in an integer register.
8977 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8978 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8979 %{
8980   match(Set dst (CmpL3 src1 src2));
8981   effect(KILL flags);
8982 
8983   ins_cost(INSN_COST * 6);
8984   format %{
8985       "cmp $src1, $src2"
8986       "csetw $dst, ne"
8987       "cnegw $dst, lt"
8988   %}
8989   // format %{ "CmpL3 $dst, $src1, $src2" %}
8990   ins_encode %{
8991     __ cmp($src1$$Register, $src2$$Register);
8992     __ csetw($dst$$Register, Assembler::NE);
8993     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8994   %}
8995 
8996   ins_pipe(pipe_class_default);
8997 %}
8998 
8999 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9000 %{
9001   match(Set dst (CmpL3 src1 src2));
9002   effect(KILL flags);
9003 
9004   ins_cost(INSN_COST * 6);
9005   format %{
9006       "cmp $src1, $src2"
9007       "csetw $dst, ne"
9008       "cnegw $dst, lt"
9009   %}
9010   ins_encode %{
9011     int32_t con = (int32_t)$src2$$constant;
9012      if (con < 0) {
9013       __ adds(zr, $src1$$Register, -con);
9014     } else {
9015       __ subs(zr, $src1$$Register, con);
9016     }
9017     __ csetw($dst$$Register, Assembler::NE);
9018     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9019   %}
9020 
9021   ins_pipe(pipe_class_default);
9022 %}
9023 
9024 // ============================================================================
9025 // Conditional Move Instructions
9026 
9027 // n.b. we have identical rules for both a signed compare op (cmpOp)
9028 // and an unsigned compare op (cmpOpU). it would be nice if we could
9029 // define an op class which merged both inputs and use it to type the
9030 // argument to a single rule. unfortunatelyt his fails because the
9031 // opclass does not live up to the COND_INTER interface of its
9032 // component operands. When the generic code tries to negate the
9033 // operand it ends up running the generci Machoper::negate method
9034 // which throws a ShouldNotHappen. So, we have to provide two flavours
9035 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9036 
9037 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9038   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9039 
9040   ins_cost(INSN_COST * 2);
9041   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
9042 
9043   ins_encode %{
9044     __ cselw(as_Register($dst$$reg),
9045              as_Register($src2$$reg),
9046              as_Register($src1$$reg),
9047              (Assembler::Condition)$cmp$$cmpcode);
9048   %}
9049 
9050   ins_pipe(icond_reg_reg);
9051 %}
9052 
9053 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9054   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9055 
9056   ins_cost(INSN_COST * 2);
9057   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
9058 
9059   ins_encode %{
9060     __ cselw(as_Register($dst$$reg),
9061              as_Register($src2$$reg),
9062              as_Register($src1$$reg),
9063              (Assembler::Condition)$cmp$$cmpcode);
9064   %}
9065 
9066   ins_pipe(icond_reg_reg);
9067 %}
9068 
9069 // special cases where one arg is zero
9070 
9071 // n.b. this is selected in preference to the rule above because it
9072 // avoids loading constant 0 into a source register
9073 
9074 // TODO
9075 // we ought only to be able to cull one of these variants as the ideal
9076 // transforms ought always to order the zero consistently (to left/right?)
9077 
9078 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9079   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9080 
9081   ins_cost(INSN_COST * 2);
9082   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
9083 
9084   ins_encode %{
9085     __ cselw(as_Register($dst$$reg),
9086              as_Register($src$$reg),
9087              zr,
9088              (Assembler::Condition)$cmp$$cmpcode);
9089   %}
9090 
9091   ins_pipe(icond_reg);
9092 %}
9093 
9094 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9095   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9096 
9097   ins_cost(INSN_COST * 2);
9098   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
9099 
9100   ins_encode %{
9101     __ cselw(as_Register($dst$$reg),
9102              as_Register($src$$reg),
9103              zr,
9104              (Assembler::Condition)$cmp$$cmpcode);
9105   %}
9106 
9107   ins_pipe(icond_reg);
9108 %}
9109 
9110 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9111   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9112 
9113   ins_cost(INSN_COST * 2);
9114   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
9115 
9116   ins_encode %{
9117     __ cselw(as_Register($dst$$reg),
9118              zr,
9119              as_Register($src$$reg),
9120              (Assembler::Condition)$cmp$$cmpcode);
9121   %}
9122 
9123   ins_pipe(icond_reg);
9124 %}
9125 
9126 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9127   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9128 
9129   ins_cost(INSN_COST * 2);
9130   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
9131 
9132   ins_encode %{
9133     __ cselw(as_Register($dst$$reg),
9134              zr,
9135              as_Register($src$$reg),
9136              (Assembler::Condition)$cmp$$cmpcode);
9137   %}
9138 
9139   ins_pipe(icond_reg);
9140 %}
9141 
9142 // special case for creating a boolean 0 or 1
9143 
9144 // n.b. this is selected in preference to the rule above because it
9145 // avoids loading constants 0 and 1 into a source register
9146 
9147 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9148   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9149 
9150   ins_cost(INSN_COST * 2);
9151   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
9152 
9153   ins_encode %{
9154     // equivalently
9155     // cset(as_Register($dst$$reg),
9156     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
9157     __ csincw(as_Register($dst$$reg),
9158              zr,
9159              zr,
9160              (Assembler::Condition)$cmp$$cmpcode);
9161   %}
9162 
9163   ins_pipe(icond_none);
9164 %}
9165 
9166 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9167   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9168 
9169   ins_cost(INSN_COST * 2);
9170   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
9171 
9172   ins_encode %{
9173     // equivalently
9174     // cset(as_Register($dst$$reg),
9175     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
9176     __ csincw(as_Register($dst$$reg),
9177              zr,
9178              zr,
9179              (Assembler::Condition)$cmp$$cmpcode);
9180   %}
9181 
9182   ins_pipe(icond_none);
9183 %}
9184 
9185 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9186   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9187 
9188   ins_cost(INSN_COST * 2);
9189   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
9190 
9191   ins_encode %{
9192     __ csel(as_Register($dst$$reg),
9193             as_Register($src2$$reg),
9194             as_Register($src1$$reg),
9195             (Assembler::Condition)$cmp$$cmpcode);
9196   %}
9197 
9198   ins_pipe(icond_reg_reg);
9199 %}
9200 
9201 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9202   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9203 
9204   ins_cost(INSN_COST * 2);
9205   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
9206 
9207   ins_encode %{
9208     __ csel(as_Register($dst$$reg),
9209             as_Register($src2$$reg),
9210             as_Register($src1$$reg),
9211             (Assembler::Condition)$cmp$$cmpcode);
9212   %}
9213 
9214   ins_pipe(icond_reg_reg);
9215 %}
9216 
9217 // special cases where one arg is zero
9218 
9219 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9220   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9221 
9222   ins_cost(INSN_COST * 2);
9223   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
9224 
9225   ins_encode %{
9226     __ csel(as_Register($dst$$reg),
9227             zr,
9228             as_Register($src$$reg),
9229             (Assembler::Condition)$cmp$$cmpcode);
9230   %}
9231 
9232   ins_pipe(icond_reg);
9233 %}
9234 
9235 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9236   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9237 
9238   ins_cost(INSN_COST * 2);
9239   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
9240 
9241   ins_encode %{
9242     __ csel(as_Register($dst$$reg),
9243             zr,
9244             as_Register($src$$reg),
9245             (Assembler::Condition)$cmp$$cmpcode);
9246   %}
9247 
9248   ins_pipe(icond_reg);
9249 %}
9250 
9251 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9252   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9253 
9254   ins_cost(INSN_COST * 2);
9255   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
9256 
9257   ins_encode %{
9258     __ csel(as_Register($dst$$reg),
9259             as_Register($src$$reg),
9260             zr,
9261             (Assembler::Condition)$cmp$$cmpcode);
9262   %}
9263 
9264   ins_pipe(icond_reg);
9265 %}
9266 
9267 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9268   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9269 
9270   ins_cost(INSN_COST * 2);
9271   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
9272 
9273   ins_encode %{
9274     __ csel(as_Register($dst$$reg),
9275             as_Register($src$$reg),
9276             zr,
9277             (Assembler::Condition)$cmp$$cmpcode);
9278   %}
9279 
9280   ins_pipe(icond_reg);
9281 %}
9282 
9283 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9284   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9285 
9286   ins_cost(INSN_COST * 2);
9287   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
9288 
9289   ins_encode %{
9290     __ csel(as_Register($dst$$reg),
9291             as_Register($src2$$reg),
9292             as_Register($src1$$reg),
9293             (Assembler::Condition)$cmp$$cmpcode);
9294   %}
9295 
9296   ins_pipe(icond_reg_reg);
9297 %}
9298 
9299 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9300   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9301 
9302   ins_cost(INSN_COST * 2);
9303   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
9304 
9305   ins_encode %{
9306     __ csel(as_Register($dst$$reg),
9307             as_Register($src2$$reg),
9308             as_Register($src1$$reg),
9309             (Assembler::Condition)$cmp$$cmpcode);
9310   %}
9311 
9312   ins_pipe(icond_reg_reg);
9313 %}
9314 
9315 // special cases where one arg is zero
9316 
9317 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9318   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9319 
9320   ins_cost(INSN_COST * 2);
9321   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
9322 
9323   ins_encode %{
9324     __ csel(as_Register($dst$$reg),
9325             zr,
9326             as_Register($src$$reg),
9327             (Assembler::Condition)$cmp$$cmpcode);
9328   %}
9329 
9330   ins_pipe(icond_reg);
9331 %}
9332 
9333 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9334   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9335 
9336   ins_cost(INSN_COST * 2);
9337   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
9338 
9339   ins_encode %{
9340     __ csel(as_Register($dst$$reg),
9341             zr,
9342             as_Register($src$$reg),
9343             (Assembler::Condition)$cmp$$cmpcode);
9344   %}
9345 
9346   ins_pipe(icond_reg);
9347 %}
9348 
9349 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9350   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9351 
9352   ins_cost(INSN_COST * 2);
9353   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
9354 
9355   ins_encode %{
9356     __ csel(as_Register($dst$$reg),
9357             as_Register($src$$reg),
9358             zr,
9359             (Assembler::Condition)$cmp$$cmpcode);
9360   %}
9361 
9362   ins_pipe(icond_reg);
9363 %}
9364 
9365 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9366   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9367 
9368   ins_cost(INSN_COST * 2);
9369   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
9370 
9371   ins_encode %{
9372     __ csel(as_Register($dst$$reg),
9373             as_Register($src$$reg),
9374             zr,
9375             (Assembler::Condition)$cmp$$cmpcode);
9376   %}
9377 
9378   ins_pipe(icond_reg);
9379 %}
9380 
9381 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9382   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9383 
9384   ins_cost(INSN_COST * 2);
9385   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
9386 
9387   ins_encode %{
9388     __ cselw(as_Register($dst$$reg),
9389              as_Register($src2$$reg),
9390              as_Register($src1$$reg),
9391              (Assembler::Condition)$cmp$$cmpcode);
9392   %}
9393 
9394   ins_pipe(icond_reg_reg);
9395 %}
9396 
9397 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9398   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9399 
9400   ins_cost(INSN_COST * 2);
9401   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
9402 
9403   ins_encode %{
9404     __ cselw(as_Register($dst$$reg),
9405              as_Register($src2$$reg),
9406              as_Register($src1$$reg),
9407              (Assembler::Condition)$cmp$$cmpcode);
9408   %}
9409 
9410   ins_pipe(icond_reg_reg);
9411 %}
9412 
9413 // special cases where one arg is zero
9414 
9415 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9416   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9417 
9418   ins_cost(INSN_COST * 2);
9419   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
9420 
9421   ins_encode %{
9422     __ cselw(as_Register($dst$$reg),
9423              zr,
9424              as_Register($src$$reg),
9425              (Assembler::Condition)$cmp$$cmpcode);
9426   %}
9427 
9428   ins_pipe(icond_reg);
9429 %}
9430 
9431 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9432   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9433 
9434   ins_cost(INSN_COST * 2);
9435   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
9436 
9437   ins_encode %{
9438     __ cselw(as_Register($dst$$reg),
9439              zr,
9440              as_Register($src$$reg),
9441              (Assembler::Condition)$cmp$$cmpcode);
9442   %}
9443 
9444   ins_pipe(icond_reg);
9445 %}
9446 
9447 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9448   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9449 
9450   ins_cost(INSN_COST * 2);
9451   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
9452 
9453   ins_encode %{
9454     __ cselw(as_Register($dst$$reg),
9455              as_Register($src$$reg),
9456              zr,
9457              (Assembler::Condition)$cmp$$cmpcode);
9458   %}
9459 
9460   ins_pipe(icond_reg);
9461 %}
9462 
9463 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9464   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9465 
9466   ins_cost(INSN_COST * 2);
9467   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
9468 
9469   ins_encode %{
9470     __ cselw(as_Register($dst$$reg),
9471              as_Register($src$$reg),
9472              zr,
9473              (Assembler::Condition)$cmp$$cmpcode);
9474   %}
9475 
9476   ins_pipe(icond_reg);
9477 %}
9478 
9479 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
9480 %{
9481   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9482 
9483   ins_cost(INSN_COST * 3);
9484 
9485   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9486   ins_encode %{
9487     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9488     __ fcsels(as_FloatRegister($dst$$reg),
9489               as_FloatRegister($src2$$reg),
9490               as_FloatRegister($src1$$reg),
9491               cond);
9492   %}
9493 
9494   ins_pipe(fp_cond_reg_reg_s);
9495 %}
9496 
9497 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
9498 %{
9499   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9500 
9501   ins_cost(INSN_COST * 3);
9502 
9503   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9504   ins_encode %{
9505     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9506     __ fcsels(as_FloatRegister($dst$$reg),
9507               as_FloatRegister($src2$$reg),
9508               as_FloatRegister($src1$$reg),
9509               cond);
9510   %}
9511 
9512   ins_pipe(fp_cond_reg_reg_s);
9513 %}
9514 
9515 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
9516 %{
9517   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9518 
9519   ins_cost(INSN_COST * 3);
9520 
9521   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9522   ins_encode %{
9523     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9524     __ fcseld(as_FloatRegister($dst$$reg),
9525               as_FloatRegister($src2$$reg),
9526               as_FloatRegister($src1$$reg),
9527               cond);
9528   %}
9529 
9530   ins_pipe(fp_cond_reg_reg_d);
9531 %}
9532 
9533 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
9534 %{
9535   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9536 
9537   ins_cost(INSN_COST * 3);
9538 
9539   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9540   ins_encode %{
9541     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9542     __ fcseld(as_FloatRegister($dst$$reg),
9543               as_FloatRegister($src2$$reg),
9544               as_FloatRegister($src1$$reg),
9545               cond);
9546   %}
9547 
9548   ins_pipe(fp_cond_reg_reg_d);
9549 %}
9550 
9551 // ============================================================================
9552 // Arithmetic Instructions
9553 //
9554 
9555 // Integer Addition
9556 
9557 // TODO
9558 // these currently employ operations which do not set CR and hence are
9559 // not flagged as killing CR but we would like to isolate the cases
9560 // where we want to set flags from those where we don't. need to work
9561 // out how to do that.
9562 
9563 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9564   match(Set dst (AddI src1 src2));
9565 
9566   ins_cost(INSN_COST);
9567   format %{ "addw  $dst, $src1, $src2" %}
9568 
9569   ins_encode %{
9570     __ addw(as_Register($dst$$reg),
9571             as_Register($src1$$reg),
9572             as_Register($src2$$reg));
9573   %}
9574 
9575   ins_pipe(ialu_reg_reg);
9576 %}
9577 
9578 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9579   match(Set dst (AddI src1 src2));
9580 
9581   ins_cost(INSN_COST);
9582   format %{ "addw $dst, $src1, $src2" %}
9583 
9584   // use opcode to indicate that this is an add not a sub
9585   opcode(0x0);
9586 
9587   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9588 
9589   ins_pipe(ialu_reg_imm);
9590 %}
9591 
9592 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9593   match(Set dst (AddI (ConvL2I src1) src2));
9594 
9595   ins_cost(INSN_COST);
9596   format %{ "addw $dst, $src1, $src2" %}
9597 
9598   // use opcode to indicate that this is an add not a sub
9599   opcode(0x0);
9600 
9601   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9602 
9603   ins_pipe(ialu_reg_imm);
9604 %}
9605 
9606 // Pointer Addition
9607 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
9608   match(Set dst (AddP src1 src2));
9609 
9610   ins_cost(INSN_COST);
9611   format %{ "add $dst, $src1, $src2\t# ptr" %}
9612 
9613   ins_encode %{
9614     __ add(as_Register($dst$$reg),
9615            as_Register($src1$$reg),
9616            as_Register($src2$$reg));
9617   %}
9618 
9619   ins_pipe(ialu_reg_reg);
9620 %}
9621 
9622 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
9623   match(Set dst (AddP src1 (ConvI2L src2)));
9624 
9625   ins_cost(1.9 * INSN_COST);
9626   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9627 
9628   ins_encode %{
9629     __ add(as_Register($dst$$reg),
9630            as_Register($src1$$reg),
9631            as_Register($src2$$reg), ext::sxtw);
9632   %}
9633 
9634   ins_pipe(ialu_reg_reg);
9635 %}
9636 
9637 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
9638   match(Set dst (AddP src1 (LShiftL src2 scale)));
9639 
9640   ins_cost(1.9 * INSN_COST);
9641   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9642 
9643   ins_encode %{
9644     __ lea(as_Register($dst$$reg),
9645            Address(as_Register($src1$$reg), as_Register($src2$$reg),
9646                    Address::lsl($scale$$constant)));
9647   %}
9648 
9649   ins_pipe(ialu_reg_reg_shift);
9650 %}
9651 
9652 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
9653   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9654 
9655   ins_cost(1.9 * INSN_COST);
9656   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9657 
9658   ins_encode %{
9659     __ lea(as_Register($dst$$reg),
9660            Address(as_Register($src1$$reg), as_Register($src2$$reg),
9661                    Address::sxtw($scale$$constant)));
9662   %}
9663 
9664   ins_pipe(ialu_reg_reg_shift);
9665 %}
9666 
9667 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9668   match(Set dst (LShiftL (ConvI2L src) scale));
9669 
9670   ins_cost(INSN_COST);
9671   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9672 
9673   ins_encode %{
9674     __ sbfiz(as_Register($dst$$reg),
9675           as_Register($src$$reg),
9676           $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
9677   %}
9678 
9679   ins_pipe(ialu_reg_shift);
9680 %}
9681 
9682 // Pointer Immediate Addition
9683 // n.b. this needs to be more expensive than using an indirect memory
9684 // operand
9685 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
9686   match(Set dst (AddP src1 src2));
9687 
9688   ins_cost(INSN_COST);
9689   format %{ "add $dst, $src1, $src2\t# ptr" %}
9690 
9691   // use opcode to indicate that this is an add not a sub
9692   opcode(0x0);
9693 
9694   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9695 
9696   ins_pipe(ialu_reg_imm);
9697 %}
9698 
9699 // Long Addition
9700 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9701 
9702   match(Set dst (AddL src1 src2));
9703 
9704   ins_cost(INSN_COST);
9705   format %{ "add  $dst, $src1, $src2" %}
9706 
9707   ins_encode %{
9708     __ add(as_Register($dst$$reg),
9709            as_Register($src1$$reg),
9710            as_Register($src2$$reg));
9711   %}
9712 
9713   ins_pipe(ialu_reg_reg);
9714 %}
9715 
9716 // No constant pool entries requiredLong Immediate Addition.
9717 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9718   match(Set dst (AddL src1 src2));
9719 
9720   ins_cost(INSN_COST);
9721   format %{ "add $dst, $src1, $src2" %}
9722 
9723   // use opcode to indicate that this is an add not a sub
9724   opcode(0x0);
9725 
9726   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9727 
9728   ins_pipe(ialu_reg_imm);
9729 %}
9730 
9731 // Integer Subtraction
9732 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9733   match(Set dst (SubI src1 src2));
9734 
9735   ins_cost(INSN_COST);
9736   format %{ "subw  $dst, $src1, $src2" %}
9737 
9738   ins_encode %{
9739     __ subw(as_Register($dst$$reg),
9740             as_Register($src1$$reg),
9741             as_Register($src2$$reg));
9742   %}
9743 
9744   ins_pipe(ialu_reg_reg);
9745 %}
9746 
9747 // Immediate Subtraction
9748 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9749   match(Set dst (SubI src1 src2));
9750 
9751   ins_cost(INSN_COST);
9752   format %{ "subw $dst, $src1, $src2" %}
9753 
9754   // use opcode to indicate that this is a sub not an add
9755   opcode(0x1);
9756 
9757   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9758 
9759   ins_pipe(ialu_reg_imm);
9760 %}
9761 
9762 // Long Subtraction
9763 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9764 
9765   match(Set dst (SubL src1 src2));
9766 
9767   ins_cost(INSN_COST);
9768   format %{ "sub  $dst, $src1, $src2" %}
9769 
9770   ins_encode %{
9771     __ sub(as_Register($dst$$reg),
9772            as_Register($src1$$reg),
9773            as_Register($src2$$reg));
9774   %}
9775 
9776   ins_pipe(ialu_reg_reg);
9777 %}
9778 
9779 // No constant pool entries requiredLong Immediate Subtraction.
9780 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9781   match(Set dst (SubL src1 src2));
9782 
9783   ins_cost(INSN_COST);
9784   format %{ "sub$dst, $src1, $src2" %}
9785 
9786   // use opcode to indicate that this is a sub not an add
9787   opcode(0x1);
9788 
9789   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9790 
9791   ins_pipe(ialu_reg_imm);
9792 %}
9793 
9794 // Integer Negation (special case for sub)
9795 
9796 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9797   match(Set dst (SubI zero src));
9798 
9799   ins_cost(INSN_COST);
9800   format %{ "negw $dst, $src\t# int" %}
9801 
9802   ins_encode %{
9803     __ negw(as_Register($dst$$reg),
9804             as_Register($src$$reg));
9805   %}
9806 
9807   ins_pipe(ialu_reg);
9808 %}
9809 
9810 // Long Negation
9811 
9812 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9813   match(Set dst (SubL zero src));
9814 
9815   ins_cost(INSN_COST);
9816   format %{ "neg $dst, $src\t# long" %}
9817 
9818   ins_encode %{
9819     __ neg(as_Register($dst$$reg),
9820            as_Register($src$$reg));
9821   %}
9822 
9823   ins_pipe(ialu_reg);
9824 %}
9825 
9826 // Integer Multiply
9827 
9828 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9829   match(Set dst (MulI src1 src2));
9830 
9831   ins_cost(INSN_COST * 3);
9832   format %{ "mulw  $dst, $src1, $src2" %}
9833 
9834   ins_encode %{
9835     __ mulw(as_Register($dst$$reg),
9836             as_Register($src1$$reg),
9837             as_Register($src2$$reg));
9838   %}
9839 
9840   ins_pipe(imul_reg_reg);
9841 %}
9842 
9843 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9844   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9845 
9846   ins_cost(INSN_COST * 3);
9847   format %{ "smull  $dst, $src1, $src2" %}
9848 
9849   ins_encode %{
9850     __ smull(as_Register($dst$$reg),
9851              as_Register($src1$$reg),
9852              as_Register($src2$$reg));
9853   %}
9854 
9855   ins_pipe(imul_reg_reg);
9856 %}
9857 
9858 // Long Multiply
9859 
9860 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9861   match(Set dst (MulL src1 src2));
9862 
9863   ins_cost(INSN_COST * 5);
9864   format %{ "mul  $dst, $src1, $src2" %}
9865 
9866   ins_encode %{
9867     __ mul(as_Register($dst$$reg),
9868            as_Register($src1$$reg),
9869            as_Register($src2$$reg));
9870   %}
9871 
9872   ins_pipe(lmul_reg_reg);
9873 %}
9874 
9875 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9876 %{
9877   match(Set dst (MulHiL src1 src2));
9878 
9879   ins_cost(INSN_COST * 7);
9880   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
9881 
9882   ins_encode %{
9883     __ smulh(as_Register($dst$$reg),
9884              as_Register($src1$$reg),
9885              as_Register($src2$$reg));
9886   %}
9887 
9888   ins_pipe(lmul_reg_reg);
9889 %}
9890 
9891 // Combined Integer Multiply & Add/Sub
9892 
9893 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9894   match(Set dst (AddI src3 (MulI src1 src2)));
9895 
9896   ins_cost(INSN_COST * 3);
9897   format %{ "madd  $dst, $src1, $src2, $src3" %}
9898 
9899   ins_encode %{
9900     __ maddw(as_Register($dst$$reg),
9901              as_Register($src1$$reg),
9902              as_Register($src2$$reg),
9903              as_Register($src3$$reg));
9904   %}
9905 
9906   ins_pipe(imac_reg_reg);
9907 %}
9908 
9909 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9910   match(Set dst (SubI src3 (MulI src1 src2)));
9911 
9912   ins_cost(INSN_COST * 3);
9913   format %{ "msub  $dst, $src1, $src2, $src3" %}
9914 
9915   ins_encode %{
9916     __ msubw(as_Register($dst$$reg),
9917              as_Register($src1$$reg),
9918              as_Register($src2$$reg),
9919              as_Register($src3$$reg));
9920   %}
9921 
9922   ins_pipe(imac_reg_reg);
9923 %}
9924 
9925 // Combined Integer Multiply & Neg
9926 
9927 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9928   match(Set dst (MulI (SubI zero src1) src2));
9929   match(Set dst (MulI src1 (SubI zero src2)));
9930 
9931   ins_cost(INSN_COST * 3);
9932   format %{ "mneg  $dst, $src1, $src2" %}
9933 
9934   ins_encode %{
9935     __ mnegw(as_Register($dst$$reg),
9936              as_Register($src1$$reg),
9937              as_Register($src2$$reg));
9938   %}
9939 
9940   ins_pipe(imac_reg_reg);
9941 %}
9942 
9943 // Combined Long Multiply & Add/Sub
9944 
9945 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9946   match(Set dst (AddL src3 (MulL src1 src2)));
9947 
9948   ins_cost(INSN_COST * 5);
9949   format %{ "madd  $dst, $src1, $src2, $src3" %}
9950 
9951   ins_encode %{
9952     __ madd(as_Register($dst$$reg),
9953             as_Register($src1$$reg),
9954             as_Register($src2$$reg),
9955             as_Register($src3$$reg));
9956   %}
9957 
9958   ins_pipe(lmac_reg_reg);
9959 %}
9960 
9961 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9962   match(Set dst (SubL src3 (MulL src1 src2)));
9963 
9964   ins_cost(INSN_COST * 5);
9965   format %{ "msub  $dst, $src1, $src2, $src3" %}
9966 
9967   ins_encode %{
9968     __ msub(as_Register($dst$$reg),
9969             as_Register($src1$$reg),
9970             as_Register($src2$$reg),
9971             as_Register($src3$$reg));
9972   %}
9973 
9974   ins_pipe(lmac_reg_reg);
9975 %}
9976 
9977 // Combined Long Multiply & Neg
9978 
9979 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9980   match(Set dst (MulL (SubL zero src1) src2));
9981   match(Set dst (MulL src1 (SubL zero src2)));
9982 
9983   ins_cost(INSN_COST * 5);
9984   format %{ "mneg  $dst, $src1, $src2" %}
9985 
9986   ins_encode %{
9987     __ mneg(as_Register($dst$$reg),
9988             as_Register($src1$$reg),
9989             as_Register($src2$$reg));
9990   %}
9991 
9992   ins_pipe(lmac_reg_reg);
9993 %}
9994 
9995 // Integer Divide
9996 
9997 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9998   match(Set dst (DivI src1 src2));
9999 
10000   ins_cost(INSN_COST * 19);
10001   format %{ "sdivw  $dst, $src1, $src2" %}
10002 
10003   ins_encode(aarch64_enc_divw(dst, src1, src2));
10004   ins_pipe(idiv_reg_reg);
10005 %}
10006 
10007 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
10008   match(Set dst (URShiftI (RShiftI src1 div1) div2));
10009   ins_cost(INSN_COST);
10010   format %{ "lsrw $dst, $src1, $div1" %}
10011   ins_encode %{
10012     __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
10013   %}
10014   ins_pipe(ialu_reg_shift);
10015 %}
10016 
10017 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
10018   match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
10019   ins_cost(INSN_COST);
10020   format %{ "addw $dst, $src, LSR $div1" %}
10021 
10022   ins_encode %{
10023     __ addw(as_Register($dst$$reg),
10024               as_Register($src$$reg),
10025               as_Register($src$$reg),
10026               Assembler::LSR, 31);
10027   %}
10028   ins_pipe(ialu_reg);
10029 %}
10030 
10031 // Long Divide
10032 
10033 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10034   match(Set dst (DivL src1 src2));
10035 
10036   ins_cost(INSN_COST * 35);
10037   format %{ "sdiv   $dst, $src1, $src2" %}
10038 
10039   ins_encode(aarch64_enc_div(dst, src1, src2));
10040   ins_pipe(ldiv_reg_reg);
10041 %}
10042 
10043 instruct signExtractL(iRegLNoSp dst, iRegL src1, immI_63 div1, immI_63 div2) %{
10044   match(Set dst (URShiftL (RShiftL src1 div1) div2));
10045   ins_cost(INSN_COST);
10046   format %{ "lsr $dst, $src1, $div1" %}
10047   ins_encode %{
10048     __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
10049   %}
10050   ins_pipe(ialu_reg_shift);
10051 %}
10052 
10053 instruct div2RoundL(iRegLNoSp dst, iRegL src, immI_63 div1, immI_63 div2) %{
10054   match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
10055   ins_cost(INSN_COST);
10056   format %{ "add $dst, $src, $div1" %}
10057 
10058   ins_encode %{
10059     __ add(as_Register($dst$$reg),
10060               as_Register($src$$reg),
10061               as_Register($src$$reg),
10062               Assembler::LSR, 63);
10063   %}
10064   ins_pipe(ialu_reg);
10065 %}
10066 
10067 // Integer Remainder
10068 
10069 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10070   match(Set dst (ModI src1 src2));
10071 
10072   ins_cost(INSN_COST * 22);
10073   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10074             "msubw($dst, rscratch1, $src2, $src1" %}
10075 
10076   ins_encode(aarch64_enc_modw(dst, src1, src2));
10077   ins_pipe(idiv_reg_reg);
10078 %}
10079 
10080 // Long Remainder
10081 
10082 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10083   match(Set dst (ModL src1 src2));
10084 
10085   ins_cost(INSN_COST * 38);
10086   format %{ "sdiv   rscratch1, $src1, $src2\n"
10087             "msub($dst, rscratch1, $src2, $src1" %}
10088 
10089   ins_encode(aarch64_enc_mod(dst, src1, src2));
10090   ins_pipe(ldiv_reg_reg);
10091 %}
10092 
10093 // Integer Shifts
10094 
10095 // Shift Left Register
10096 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10097   match(Set dst (LShiftI src1 src2));
10098 
10099   ins_cost(INSN_COST * 2);
10100   format %{ "lslvw  $dst, $src1, $src2" %}
10101 
10102   ins_encode %{
10103     __ lslvw(as_Register($dst$$reg),
10104              as_Register($src1$$reg),
10105              as_Register($src2$$reg));
10106   %}
10107 
10108   ins_pipe(ialu_reg_reg_vshift);
10109 %}
10110 
10111 // Shift Left Immediate
10112 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10113   match(Set dst (LShiftI src1 src2));
10114 
10115   ins_cost(INSN_COST);
10116   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10117 
10118   ins_encode %{
10119     __ lslw(as_Register($dst$$reg),
10120             as_Register($src1$$reg),
10121             $src2$$constant & 0x1f);
10122   %}
10123 
10124   ins_pipe(ialu_reg_shift);
10125 %}
10126 
10127 // Shift Right Logical Register
10128 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10129   match(Set dst (URShiftI src1 src2));
10130 
10131   ins_cost(INSN_COST * 2);
10132   format %{ "lsrvw  $dst, $src1, $src2" %}
10133 
10134   ins_encode %{
10135     __ lsrvw(as_Register($dst$$reg),
10136              as_Register($src1$$reg),
10137              as_Register($src2$$reg));
10138   %}
10139 
10140   ins_pipe(ialu_reg_reg_vshift);
10141 %}
10142 
10143 // Shift Right Logical Immediate
10144 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10145   match(Set dst (URShiftI src1 src2));
10146 
10147   ins_cost(INSN_COST);
10148   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10149 
10150   ins_encode %{
10151     __ lsrw(as_Register($dst$$reg),
10152             as_Register($src1$$reg),
10153             $src2$$constant & 0x1f);
10154   %}
10155 
10156   ins_pipe(ialu_reg_shift);
10157 %}
10158 
10159 // Shift Right Arithmetic Register
10160 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10161   match(Set dst (RShiftI src1 src2));
10162 
10163   ins_cost(INSN_COST * 2);
10164   format %{ "asrvw  $dst, $src1, $src2" %}
10165 
10166   ins_encode %{
10167     __ asrvw(as_Register($dst$$reg),
10168              as_Register($src1$$reg),
10169              as_Register($src2$$reg));
10170   %}
10171 
10172   ins_pipe(ialu_reg_reg_vshift);
10173 %}
10174 
10175 // Shift Right Arithmetic Immediate
10176 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10177   match(Set dst (RShiftI src1 src2));
10178 
10179   ins_cost(INSN_COST);
10180   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10181 
10182   ins_encode %{
10183     __ asrw(as_Register($dst$$reg),
10184             as_Register($src1$$reg),
10185             $src2$$constant & 0x1f);
10186   %}
10187 
10188   ins_pipe(ialu_reg_shift);
10189 %}
10190 
10191 // Combined Int Mask and Right Shift (using UBFM)
10192 // TODO
10193 
10194 // Long Shifts
10195 
10196 // Shift Left Register
10197 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10198   match(Set dst (LShiftL src1 src2));
10199 
10200   ins_cost(INSN_COST * 2);
10201   format %{ "lslv  $dst, $src1, $src2" %}
10202 
10203   ins_encode %{
10204     __ lslv(as_Register($dst$$reg),
10205             as_Register($src1$$reg),
10206             as_Register($src2$$reg));
10207   %}
10208 
10209   ins_pipe(ialu_reg_reg_vshift);
10210 %}
10211 
10212 // Shift Left Immediate
10213 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10214   match(Set dst (LShiftL src1 src2));
10215 
10216   ins_cost(INSN_COST);
10217   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10218 
10219   ins_encode %{
10220     __ lsl(as_Register($dst$$reg),
10221             as_Register($src1$$reg),
10222             $src2$$constant & 0x3f);
10223   %}
10224 
10225   ins_pipe(ialu_reg_shift);
10226 %}
10227 
10228 // Shift Right Logical Register
10229 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10230   match(Set dst (URShiftL src1 src2));
10231 
10232   ins_cost(INSN_COST * 2);
10233   format %{ "lsrv  $dst, $src1, $src2" %}
10234 
10235   ins_encode %{
10236     __ lsrv(as_Register($dst$$reg),
10237             as_Register($src1$$reg),
10238             as_Register($src2$$reg));
10239   %}
10240 
10241   ins_pipe(ialu_reg_reg_vshift);
10242 %}
10243 
10244 // Shift Right Logical Immediate
10245 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10246   match(Set dst (URShiftL src1 src2));
10247 
10248   ins_cost(INSN_COST);
10249   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10250 
10251   ins_encode %{
10252     __ lsr(as_Register($dst$$reg),
10253            as_Register($src1$$reg),
10254            $src2$$constant & 0x3f);
10255   %}
10256 
10257   ins_pipe(ialu_reg_shift);
10258 %}
10259 
10260 // A special-case pattern for card table stores.
10261 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10262   match(Set dst (URShiftL (CastP2X src1) src2));
10263 
10264   ins_cost(INSN_COST);
10265   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10266 
10267   ins_encode %{
10268     __ lsr(as_Register($dst$$reg),
10269            as_Register($src1$$reg),
10270            $src2$$constant & 0x3f);
10271   %}
10272 
10273   ins_pipe(ialu_reg_shift);
10274 %}
10275 
10276 // Shift Right Arithmetic Register
10277 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10278   match(Set dst (RShiftL src1 src2));
10279 
10280   ins_cost(INSN_COST * 2);
10281   format %{ "asrv  $dst, $src1, $src2" %}
10282 
10283   ins_encode %{
10284     __ asrv(as_Register($dst$$reg),
10285             as_Register($src1$$reg),
10286             as_Register($src2$$reg));
10287   %}
10288 
10289   ins_pipe(ialu_reg_reg_vshift);
10290 %}
10291 
10292 // Shift Right Arithmetic Immediate
10293 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10294   match(Set dst (RShiftL src1 src2));
10295 
10296   ins_cost(INSN_COST);
10297   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10298 
10299   ins_encode %{
10300     __ asr(as_Register($dst$$reg),
10301            as_Register($src1$$reg),
10302            $src2$$constant & 0x3f);
10303   %}
10304 
10305   ins_pipe(ialu_reg_shift);
10306 %}
10307 
10308 // BEGIN This section of the file is automatically generated. Do not edit --------------
10309 
10310 instruct regL_not_reg(iRegLNoSp dst,
10311                          iRegL src1, immL_M1 m1,
10312                          rFlagsReg cr) %{
10313   match(Set dst (XorL src1 m1));
10314   ins_cost(INSN_COST);
10315   format %{ "eon  $dst, $src1, zr" %}
10316 
10317   ins_encode %{
10318     __ eon(as_Register($dst$$reg),
10319               as_Register($src1$$reg),
10320               zr,
10321               Assembler::LSL, 0);
10322   %}
10323 
10324   ins_pipe(ialu_reg);
10325 %}
10326 instruct regI_not_reg(iRegINoSp dst,
10327                          iRegIorL2I src1, immI_M1 m1,
10328                          rFlagsReg cr) %{
10329   match(Set dst (XorI src1 m1));
10330   ins_cost(INSN_COST);
10331   format %{ "eonw  $dst, $src1, zr" %}
10332 
10333   ins_encode %{
10334     __ eonw(as_Register($dst$$reg),
10335               as_Register($src1$$reg),
10336               zr,
10337               Assembler::LSL, 0);
10338   %}
10339 
10340   ins_pipe(ialu_reg);
10341 %}
10342 
10343 instruct AndI_reg_not_reg(iRegINoSp dst,
10344                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10345                          rFlagsReg cr) %{
10346   match(Set dst (AndI src1 (XorI src2 m1)));
10347   ins_cost(INSN_COST);
10348   format %{ "bicw  $dst, $src1, $src2" %}
10349 
10350   ins_encode %{
10351     __ bicw(as_Register($dst$$reg),
10352               as_Register($src1$$reg),
10353               as_Register($src2$$reg),
10354               Assembler::LSL, 0);
10355   %}
10356 
10357   ins_pipe(ialu_reg_reg);
10358 %}
10359 
10360 instruct AndL_reg_not_reg(iRegLNoSp dst,
10361                          iRegL src1, iRegL src2, immL_M1 m1,
10362                          rFlagsReg cr) %{
10363   match(Set dst (AndL src1 (XorL src2 m1)));
10364   ins_cost(INSN_COST);
10365   format %{ "bic  $dst, $src1, $src2" %}
10366 
10367   ins_encode %{
10368     __ bic(as_Register($dst$$reg),
10369               as_Register($src1$$reg),
10370               as_Register($src2$$reg),
10371               Assembler::LSL, 0);
10372   %}
10373 
10374   ins_pipe(ialu_reg_reg);
10375 %}
10376 
10377 instruct OrI_reg_not_reg(iRegINoSp dst,
10378                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10379                          rFlagsReg cr) %{
10380   match(Set dst (OrI src1 (XorI src2 m1)));
10381   ins_cost(INSN_COST);
10382   format %{ "ornw  $dst, $src1, $src2" %}
10383 
10384   ins_encode %{
10385     __ ornw(as_Register($dst$$reg),
10386               as_Register($src1$$reg),
10387               as_Register($src2$$reg),
10388               Assembler::LSL, 0);
10389   %}
10390 
10391   ins_pipe(ialu_reg_reg);
10392 %}
10393 
10394 instruct OrL_reg_not_reg(iRegLNoSp dst,
10395                          iRegL src1, iRegL src2, immL_M1 m1,
10396                          rFlagsReg cr) %{
10397   match(Set dst (OrL src1 (XorL src2 m1)));
10398   ins_cost(INSN_COST);
10399   format %{ "orn  $dst, $src1, $src2" %}
10400 
10401   ins_encode %{
10402     __ orn(as_Register($dst$$reg),
10403               as_Register($src1$$reg),
10404               as_Register($src2$$reg),
10405               Assembler::LSL, 0);
10406   %}
10407 
10408   ins_pipe(ialu_reg_reg);
10409 %}
10410 
10411 instruct XorI_reg_not_reg(iRegINoSp dst,
10412                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
10413                          rFlagsReg cr) %{
10414   match(Set dst (XorI m1 (XorI src2 src1)));
10415   ins_cost(INSN_COST);
10416   format %{ "eonw  $dst, $src1, $src2" %}
10417 
10418   ins_encode %{
10419     __ eonw(as_Register($dst$$reg),
10420               as_Register($src1$$reg),
10421               as_Register($src2$$reg),
10422               Assembler::LSL, 0);
10423   %}
10424 
10425   ins_pipe(ialu_reg_reg);
10426 %}
10427 
10428 instruct XorL_reg_not_reg(iRegLNoSp dst,
10429                          iRegL src1, iRegL src2, immL_M1 m1,
10430                          rFlagsReg cr) %{
10431   match(Set dst (XorL m1 (XorL src2 src1)));
10432   ins_cost(INSN_COST);
10433   format %{ "eon  $dst, $src1, $src2" %}
10434 
10435   ins_encode %{
10436     __ eon(as_Register($dst$$reg),
10437               as_Register($src1$$reg),
10438               as_Register($src2$$reg),
10439               Assembler::LSL, 0);
10440   %}
10441 
10442   ins_pipe(ialu_reg_reg);
10443 %}
10444 
10445 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10446                          iRegIorL2I src1, iRegIorL2I src2,
10447                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10448   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10449   ins_cost(1.9 * INSN_COST);
10450   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
10451 
10452   ins_encode %{
10453     __ bicw(as_Register($dst$$reg),
10454               as_Register($src1$$reg),
10455               as_Register($src2$$reg),
10456               Assembler::LSR,
10457               $src3$$constant & 0x1f);
10458   %}
10459 
10460   ins_pipe(ialu_reg_reg_shift);
10461 %}
10462 
10463 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10464                          iRegL src1, iRegL src2,
10465                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10466   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10467   ins_cost(1.9 * INSN_COST);
10468   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
10469 
10470   ins_encode %{
10471     __ bic(as_Register($dst$$reg),
10472               as_Register($src1$$reg),
10473               as_Register($src2$$reg),
10474               Assembler::LSR,
10475               $src3$$constant & 0x3f);
10476   %}
10477 
10478   ins_pipe(ialu_reg_reg_shift);
10479 %}
10480 
10481 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10482                          iRegIorL2I src1, iRegIorL2I src2,
10483                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10484   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10485   ins_cost(1.9 * INSN_COST);
10486   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
10487 
10488   ins_encode %{
10489     __ bicw(as_Register($dst$$reg),
10490               as_Register($src1$$reg),
10491               as_Register($src2$$reg),
10492               Assembler::ASR,
10493               $src3$$constant & 0x1f);
10494   %}
10495 
10496   ins_pipe(ialu_reg_reg_shift);
10497 %}
10498 
10499 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10500                          iRegL src1, iRegL src2,
10501                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10502   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10503   ins_cost(1.9 * INSN_COST);
10504   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
10505 
10506   ins_encode %{
10507     __ bic(as_Register($dst$$reg),
10508               as_Register($src1$$reg),
10509               as_Register($src2$$reg),
10510               Assembler::ASR,
10511               $src3$$constant & 0x3f);
10512   %}
10513 
10514   ins_pipe(ialu_reg_reg_shift);
10515 %}
10516 
10517 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10518                          iRegIorL2I src1, iRegIorL2I src2,
10519                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10520   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10521   ins_cost(1.9 * INSN_COST);
10522   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
10523 
10524   ins_encode %{
10525     __ bicw(as_Register($dst$$reg),
10526               as_Register($src1$$reg),
10527               as_Register($src2$$reg),
10528               Assembler::LSL,
10529               $src3$$constant & 0x1f);
10530   %}
10531 
10532   ins_pipe(ialu_reg_reg_shift);
10533 %}
10534 
10535 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10536                          iRegL src1, iRegL src2,
10537                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10538   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10539   ins_cost(1.9 * INSN_COST);
10540   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
10541 
10542   ins_encode %{
10543     __ bic(as_Register($dst$$reg),
10544               as_Register($src1$$reg),
10545               as_Register($src2$$reg),
10546               Assembler::LSL,
10547               $src3$$constant & 0x3f);
10548   %}
10549 
10550   ins_pipe(ialu_reg_reg_shift);
10551 %}
10552 
10553 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10554                          iRegIorL2I src1, iRegIorL2I src2,
10555                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10556   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10557   ins_cost(1.9 * INSN_COST);
10558   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
10559 
10560   ins_encode %{
10561     __ eonw(as_Register($dst$$reg),
10562               as_Register($src1$$reg),
10563               as_Register($src2$$reg),
10564               Assembler::LSR,
10565               $src3$$constant & 0x1f);
10566   %}
10567 
10568   ins_pipe(ialu_reg_reg_shift);
10569 %}
10570 
10571 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10572                          iRegL src1, iRegL src2,
10573                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10574   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10575   ins_cost(1.9 * INSN_COST);
10576   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
10577 
10578   ins_encode %{
10579     __ eon(as_Register($dst$$reg),
10580               as_Register($src1$$reg),
10581               as_Register($src2$$reg),
10582               Assembler::LSR,
10583               $src3$$constant & 0x3f);
10584   %}
10585 
10586   ins_pipe(ialu_reg_reg_shift);
10587 %}
10588 
10589 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10590                          iRegIorL2I src1, iRegIorL2I src2,
10591                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10592   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10593   ins_cost(1.9 * INSN_COST);
10594   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
10595 
10596   ins_encode %{
10597     __ eonw(as_Register($dst$$reg),
10598               as_Register($src1$$reg),
10599               as_Register($src2$$reg),
10600               Assembler::ASR,
10601               $src3$$constant & 0x1f);
10602   %}
10603 
10604   ins_pipe(ialu_reg_reg_shift);
10605 %}
10606 
10607 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10608                          iRegL src1, iRegL src2,
10609                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10610   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10611   ins_cost(1.9 * INSN_COST);
10612   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
10613 
10614   ins_encode %{
10615     __ eon(as_Register($dst$$reg),
10616               as_Register($src1$$reg),
10617               as_Register($src2$$reg),
10618               Assembler::ASR,
10619               $src3$$constant & 0x3f);
10620   %}
10621 
10622   ins_pipe(ialu_reg_reg_shift);
10623 %}
10624 
10625 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10626                          iRegIorL2I src1, iRegIorL2I src2,
10627                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10628   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10629   ins_cost(1.9 * INSN_COST);
10630   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
10631 
10632   ins_encode %{
10633     __ eonw(as_Register($dst$$reg),
10634               as_Register($src1$$reg),
10635               as_Register($src2$$reg),
10636               Assembler::LSL,
10637               $src3$$constant & 0x1f);
10638   %}
10639 
10640   ins_pipe(ialu_reg_reg_shift);
10641 %}
10642 
10643 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10644                          iRegL src1, iRegL src2,
10645                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10646   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10647   ins_cost(1.9 * INSN_COST);
10648   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
10649 
10650   ins_encode %{
10651     __ eon(as_Register($dst$$reg),
10652               as_Register($src1$$reg),
10653               as_Register($src2$$reg),
10654               Assembler::LSL,
10655               $src3$$constant & 0x3f);
10656   %}
10657 
10658   ins_pipe(ialu_reg_reg_shift);
10659 %}
10660 
10661 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10662                          iRegIorL2I src1, iRegIorL2I src2,
10663                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10664   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10665   ins_cost(1.9 * INSN_COST);
10666   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
10667 
10668   ins_encode %{
10669     __ ornw(as_Register($dst$$reg),
10670               as_Register($src1$$reg),
10671               as_Register($src2$$reg),
10672               Assembler::LSR,
10673               $src3$$constant & 0x1f);
10674   %}
10675 
10676   ins_pipe(ialu_reg_reg_shift);
10677 %}
10678 
10679 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10680                          iRegL src1, iRegL src2,
10681                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10682   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10683   ins_cost(1.9 * INSN_COST);
10684   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
10685 
10686   ins_encode %{
10687     __ orn(as_Register($dst$$reg),
10688               as_Register($src1$$reg),
10689               as_Register($src2$$reg),
10690               Assembler::LSR,
10691               $src3$$constant & 0x3f);
10692   %}
10693 
10694   ins_pipe(ialu_reg_reg_shift);
10695 %}
10696 
10697 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10698                          iRegIorL2I src1, iRegIorL2I src2,
10699                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10700   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10701   ins_cost(1.9 * INSN_COST);
10702   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
10703 
10704   ins_encode %{
10705     __ ornw(as_Register($dst$$reg),
10706               as_Register($src1$$reg),
10707               as_Register($src2$$reg),
10708               Assembler::ASR,
10709               $src3$$constant & 0x1f);
10710   %}
10711 
10712   ins_pipe(ialu_reg_reg_shift);
10713 %}
10714 
10715 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10716                          iRegL src1, iRegL src2,
10717                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10718   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10719   ins_cost(1.9 * INSN_COST);
10720   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
10721 
10722   ins_encode %{
10723     __ orn(as_Register($dst$$reg),
10724               as_Register($src1$$reg),
10725               as_Register($src2$$reg),
10726               Assembler::ASR,
10727               $src3$$constant & 0x3f);
10728   %}
10729 
10730   ins_pipe(ialu_reg_reg_shift);
10731 %}
10732 
10733 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10734                          iRegIorL2I src1, iRegIorL2I src2,
10735                          immI src3, immI_M1 src4, rFlagsReg cr) %{
10736   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10737   ins_cost(1.9 * INSN_COST);
10738   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
10739 
10740   ins_encode %{
10741     __ ornw(as_Register($dst$$reg),
10742               as_Register($src1$$reg),
10743               as_Register($src2$$reg),
10744               Assembler::LSL,
10745               $src3$$constant & 0x1f);
10746   %}
10747 
10748   ins_pipe(ialu_reg_reg_shift);
10749 %}
10750 
10751 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10752                          iRegL src1, iRegL src2,
10753                          immI src3, immL_M1 src4, rFlagsReg cr) %{
10754   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10755   ins_cost(1.9 * INSN_COST);
10756   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
10757 
10758   ins_encode %{
10759     __ orn(as_Register($dst$$reg),
10760               as_Register($src1$$reg),
10761               as_Register($src2$$reg),
10762               Assembler::LSL,
10763               $src3$$constant & 0x3f);
10764   %}
10765 
10766   ins_pipe(ialu_reg_reg_shift);
10767 %}
10768 
10769 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10770                          iRegIorL2I src1, iRegIorL2I src2,
10771                          immI src3, rFlagsReg cr) %{
10772   match(Set dst (AndI src1 (URShiftI src2 src3)));
10773 
10774   ins_cost(1.9 * INSN_COST);
10775   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
10776 
10777   ins_encode %{
10778     __ andw(as_Register($dst$$reg),
10779               as_Register($src1$$reg),
10780               as_Register($src2$$reg),
10781               Assembler::LSR,
10782               $src3$$constant & 0x1f);
10783   %}
10784 
10785   ins_pipe(ialu_reg_reg_shift);
10786 %}
10787 
10788 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10789                          iRegL src1, iRegL src2,
10790                          immI src3, rFlagsReg cr) %{
10791   match(Set dst (AndL src1 (URShiftL src2 src3)));
10792 
10793   ins_cost(1.9 * INSN_COST);
10794   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
10795 
10796   ins_encode %{
10797     __ andr(as_Register($dst$$reg),
10798               as_Register($src1$$reg),
10799               as_Register($src2$$reg),
10800               Assembler::LSR,
10801               $src3$$constant & 0x3f);
10802   %}
10803 
10804   ins_pipe(ialu_reg_reg_shift);
10805 %}
10806 
10807 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10808                          iRegIorL2I src1, iRegIorL2I src2,
10809                          immI src3, rFlagsReg cr) %{
10810   match(Set dst (AndI src1 (RShiftI src2 src3)));
10811 
10812   ins_cost(1.9 * INSN_COST);
10813   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
10814 
10815   ins_encode %{
10816     __ andw(as_Register($dst$$reg),
10817               as_Register($src1$$reg),
10818               as_Register($src2$$reg),
10819               Assembler::ASR,
10820               $src3$$constant & 0x1f);
10821   %}
10822 
10823   ins_pipe(ialu_reg_reg_shift);
10824 %}
10825 
10826 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10827                          iRegL src1, iRegL src2,
10828                          immI src3, rFlagsReg cr) %{
10829   match(Set dst (AndL src1 (RShiftL src2 src3)));
10830 
10831   ins_cost(1.9 * INSN_COST);
10832   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
10833 
10834   ins_encode %{
10835     __ andr(as_Register($dst$$reg),
10836               as_Register($src1$$reg),
10837               as_Register($src2$$reg),
10838               Assembler::ASR,
10839               $src3$$constant & 0x3f);
10840   %}
10841 
10842   ins_pipe(ialu_reg_reg_shift);
10843 %}
10844 
10845 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10846                          iRegIorL2I src1, iRegIorL2I src2,
10847                          immI src3, rFlagsReg cr) %{
10848   match(Set dst (AndI src1 (LShiftI src2 src3)));
10849 
10850   ins_cost(1.9 * INSN_COST);
10851   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
10852 
10853   ins_encode %{
10854     __ andw(as_Register($dst$$reg),
10855               as_Register($src1$$reg),
10856               as_Register($src2$$reg),
10857               Assembler::LSL,
10858               $src3$$constant & 0x1f);
10859   %}
10860 
10861   ins_pipe(ialu_reg_reg_shift);
10862 %}
10863 
10864 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10865                          iRegL src1, iRegL src2,
10866                          immI src3, rFlagsReg cr) %{
10867   match(Set dst (AndL src1 (LShiftL src2 src3)));
10868 
10869   ins_cost(1.9 * INSN_COST);
10870   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
10871 
10872   ins_encode %{
10873     __ andr(as_Register($dst$$reg),
10874               as_Register($src1$$reg),
10875               as_Register($src2$$reg),
10876               Assembler::LSL,
10877               $src3$$constant & 0x3f);
10878   %}
10879 
10880   ins_pipe(ialu_reg_reg_shift);
10881 %}
10882 
10883 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10884                          iRegIorL2I src1, iRegIorL2I src2,
10885                          immI src3, rFlagsReg cr) %{
10886   match(Set dst (XorI src1 (URShiftI src2 src3)));
10887 
10888   ins_cost(1.9 * INSN_COST);
10889   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
10890 
10891   ins_encode %{
10892     __ eorw(as_Register($dst$$reg),
10893               as_Register($src1$$reg),
10894               as_Register($src2$$reg),
10895               Assembler::LSR,
10896               $src3$$constant & 0x1f);
10897   %}
10898 
10899   ins_pipe(ialu_reg_reg_shift);
10900 %}
10901 
10902 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10903                          iRegL src1, iRegL src2,
10904                          immI src3, rFlagsReg cr) %{
10905   match(Set dst (XorL src1 (URShiftL src2 src3)));
10906 
10907   ins_cost(1.9 * INSN_COST);
10908   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
10909 
10910   ins_encode %{
10911     __ eor(as_Register($dst$$reg),
10912               as_Register($src1$$reg),
10913               as_Register($src2$$reg),
10914               Assembler::LSR,
10915               $src3$$constant & 0x3f);
10916   %}
10917 
10918   ins_pipe(ialu_reg_reg_shift);
10919 %}
10920 
10921 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10922                          iRegIorL2I src1, iRegIorL2I src2,
10923                          immI src3, rFlagsReg cr) %{
10924   match(Set dst (XorI src1 (RShiftI src2 src3)));
10925 
10926   ins_cost(1.9 * INSN_COST);
10927   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
10928 
10929   ins_encode %{
10930     __ eorw(as_Register($dst$$reg),
10931               as_Register($src1$$reg),
10932               as_Register($src2$$reg),
10933               Assembler::ASR,
10934               $src3$$constant & 0x1f);
10935   %}
10936 
10937   ins_pipe(ialu_reg_reg_shift);
10938 %}
10939 
10940 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10941                          iRegL src1, iRegL src2,
10942                          immI src3, rFlagsReg cr) %{
10943   match(Set dst (XorL src1 (RShiftL src2 src3)));
10944 
10945   ins_cost(1.9 * INSN_COST);
10946   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
10947 
10948   ins_encode %{
10949     __ eor(as_Register($dst$$reg),
10950               as_Register($src1$$reg),
10951               as_Register($src2$$reg),
10952               Assembler::ASR,
10953               $src3$$constant & 0x3f);
10954   %}
10955 
10956   ins_pipe(ialu_reg_reg_shift);
10957 %}
10958 
10959 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10960                          iRegIorL2I src1, iRegIorL2I src2,
10961                          immI src3, rFlagsReg cr) %{
10962   match(Set dst (XorI src1 (LShiftI src2 src3)));
10963 
10964   ins_cost(1.9 * INSN_COST);
10965   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
10966 
10967   ins_encode %{
10968     __ eorw(as_Register($dst$$reg),
10969               as_Register($src1$$reg),
10970               as_Register($src2$$reg),
10971               Assembler::LSL,
10972               $src3$$constant & 0x1f);
10973   %}
10974 
10975   ins_pipe(ialu_reg_reg_shift);
10976 %}
10977 
10978 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10979                          iRegL src1, iRegL src2,
10980                          immI src3, rFlagsReg cr) %{
10981   match(Set dst (XorL src1 (LShiftL src2 src3)));
10982 
10983   ins_cost(1.9 * INSN_COST);
10984   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
10985 
10986   ins_encode %{
10987     __ eor(as_Register($dst$$reg),
10988               as_Register($src1$$reg),
10989               as_Register($src2$$reg),
10990               Assembler::LSL,
10991               $src3$$constant & 0x3f);
10992   %}
10993 
10994   ins_pipe(ialu_reg_reg_shift);
10995 %}
10996 
10997 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10998                          iRegIorL2I src1, iRegIorL2I src2,
10999                          immI src3, rFlagsReg cr) %{
11000   match(Set dst (OrI src1 (URShiftI src2 src3)));
11001 
11002   ins_cost(1.9 * INSN_COST);
11003   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11004 
11005   ins_encode %{
11006     __ orrw(as_Register($dst$$reg),
11007               as_Register($src1$$reg),
11008               as_Register($src2$$reg),
11009               Assembler::LSR,
11010               $src3$$constant & 0x1f);
11011   %}
11012 
11013   ins_pipe(ialu_reg_reg_shift);
11014 %}
11015 
11016 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11017                          iRegL src1, iRegL src2,
11018                          immI src3, rFlagsReg cr) %{
11019   match(Set dst (OrL src1 (URShiftL src2 src3)));
11020 
11021   ins_cost(1.9 * INSN_COST);
11022   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11023 
11024   ins_encode %{
11025     __ orr(as_Register($dst$$reg),
11026               as_Register($src1$$reg),
11027               as_Register($src2$$reg),
11028               Assembler::LSR,
11029               $src3$$constant & 0x3f);
11030   %}
11031 
11032   ins_pipe(ialu_reg_reg_shift);
11033 %}
11034 
11035 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11036                          iRegIorL2I src1, iRegIorL2I src2,
11037                          immI src3, rFlagsReg cr) %{
11038   match(Set dst (OrI src1 (RShiftI src2 src3)));
11039 
11040   ins_cost(1.9 * INSN_COST);
11041   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
11042 
11043   ins_encode %{
11044     __ orrw(as_Register($dst$$reg),
11045               as_Register($src1$$reg),
11046               as_Register($src2$$reg),
11047               Assembler::ASR,
11048               $src3$$constant & 0x1f);
11049   %}
11050 
11051   ins_pipe(ialu_reg_reg_shift);
11052 %}
11053 
11054 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11055                          iRegL src1, iRegL src2,
11056                          immI src3, rFlagsReg cr) %{
11057   match(Set dst (OrL src1 (RShiftL src2 src3)));
11058 
11059   ins_cost(1.9 * INSN_COST);
11060   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
11061 
11062   ins_encode %{
11063     __ orr(as_Register($dst$$reg),
11064               as_Register($src1$$reg),
11065               as_Register($src2$$reg),
11066               Assembler::ASR,
11067               $src3$$constant & 0x3f);
11068   %}
11069 
11070   ins_pipe(ialu_reg_reg_shift);
11071 %}
11072 
11073 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11074                          iRegIorL2I src1, iRegIorL2I src2,
11075                          immI src3, rFlagsReg cr) %{
11076   match(Set dst (OrI src1 (LShiftI src2 src3)));
11077 
11078   ins_cost(1.9 * INSN_COST);
11079   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
11080 
11081   ins_encode %{
11082     __ orrw(as_Register($dst$$reg),
11083               as_Register($src1$$reg),
11084               as_Register($src2$$reg),
11085               Assembler::LSL,
11086               $src3$$constant & 0x1f);
11087   %}
11088 
11089   ins_pipe(ialu_reg_reg_shift);
11090 %}
11091 
11092 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11093                          iRegL src1, iRegL src2,
11094                          immI src3, rFlagsReg cr) %{
11095   match(Set dst (OrL src1 (LShiftL src2 src3)));
11096 
11097   ins_cost(1.9 * INSN_COST);
11098   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
11099 
11100   ins_encode %{
11101     __ orr(as_Register($dst$$reg),
11102               as_Register($src1$$reg),
11103               as_Register($src2$$reg),
11104               Assembler::LSL,
11105               $src3$$constant & 0x3f);
11106   %}
11107 
11108   ins_pipe(ialu_reg_reg_shift);
11109 %}
11110 
11111 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11112                          iRegIorL2I src1, iRegIorL2I src2,
11113                          immI src3, rFlagsReg cr) %{
11114   match(Set dst (AddI src1 (URShiftI src2 src3)));
11115 
11116   ins_cost(1.9 * INSN_COST);
11117   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
11118 
11119   ins_encode %{
11120     __ addw(as_Register($dst$$reg),
11121               as_Register($src1$$reg),
11122               as_Register($src2$$reg),
11123               Assembler::LSR,
11124               $src3$$constant & 0x1f);
11125   %}
11126 
11127   ins_pipe(ialu_reg_reg_shift);
11128 %}
11129 
11130 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11131                          iRegL src1, iRegL src2,
11132                          immI src3, rFlagsReg cr) %{
11133   match(Set dst (AddL src1 (URShiftL src2 src3)));
11134 
11135   ins_cost(1.9 * INSN_COST);
11136   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
11137 
11138   ins_encode %{
11139     __ add(as_Register($dst$$reg),
11140               as_Register($src1$$reg),
11141               as_Register($src2$$reg),
11142               Assembler::LSR,
11143               $src3$$constant & 0x3f);
11144   %}
11145 
11146   ins_pipe(ialu_reg_reg_shift);
11147 %}
11148 
11149 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11150                          iRegIorL2I src1, iRegIorL2I src2,
11151                          immI src3, rFlagsReg cr) %{
11152   match(Set dst (AddI src1 (RShiftI src2 src3)));
11153 
11154   ins_cost(1.9 * INSN_COST);
11155   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
11156 
11157   ins_encode %{
11158     __ addw(as_Register($dst$$reg),
11159               as_Register($src1$$reg),
11160               as_Register($src2$$reg),
11161               Assembler::ASR,
11162               $src3$$constant & 0x1f);
11163   %}
11164 
11165   ins_pipe(ialu_reg_reg_shift);
11166 %}
11167 
11168 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11169                          iRegL src1, iRegL src2,
11170                          immI src3, rFlagsReg cr) %{
11171   match(Set dst (AddL src1 (RShiftL src2 src3)));
11172 
11173   ins_cost(1.9 * INSN_COST);
11174   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
11175 
11176   ins_encode %{
11177     __ add(as_Register($dst$$reg),
11178               as_Register($src1$$reg),
11179               as_Register($src2$$reg),
11180               Assembler::ASR,
11181               $src3$$constant & 0x3f);
11182   %}
11183 
11184   ins_pipe(ialu_reg_reg_shift);
11185 %}
11186 
11187 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11188                          iRegIorL2I src1, iRegIorL2I src2,
11189                          immI src3, rFlagsReg cr) %{
11190   match(Set dst (AddI src1 (LShiftI src2 src3)));
11191 
11192   ins_cost(1.9 * INSN_COST);
11193   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
11194 
11195   ins_encode %{
11196     __ addw(as_Register($dst$$reg),
11197               as_Register($src1$$reg),
11198               as_Register($src2$$reg),
11199               Assembler::LSL,
11200               $src3$$constant & 0x1f);
11201   %}
11202 
11203   ins_pipe(ialu_reg_reg_shift);
11204 %}
11205 
11206 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11207                          iRegL src1, iRegL src2,
11208                          immI src3, rFlagsReg cr) %{
11209   match(Set dst (AddL src1 (LShiftL src2 src3)));
11210 
11211   ins_cost(1.9 * INSN_COST);
11212   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
11213 
11214   ins_encode %{
11215     __ add(as_Register($dst$$reg),
11216               as_Register($src1$$reg),
11217               as_Register($src2$$reg),
11218               Assembler::LSL,
11219               $src3$$constant & 0x3f);
11220   %}
11221 
11222   ins_pipe(ialu_reg_reg_shift);
11223 %}
11224 
11225 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11226                          iRegIorL2I src1, iRegIorL2I src2,
11227                          immI src3, rFlagsReg cr) %{
11228   match(Set dst (SubI src1 (URShiftI src2 src3)));
11229 
11230   ins_cost(1.9 * INSN_COST);
11231   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
11232 
11233   ins_encode %{
11234     __ subw(as_Register($dst$$reg),
11235               as_Register($src1$$reg),
11236               as_Register($src2$$reg),
11237               Assembler::LSR,
11238               $src3$$constant & 0x1f);
11239   %}
11240 
11241   ins_pipe(ialu_reg_reg_shift);
11242 %}
11243 
11244 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11245                          iRegL src1, iRegL src2,
11246                          immI src3, rFlagsReg cr) %{
11247   match(Set dst (SubL src1 (URShiftL src2 src3)));
11248 
11249   ins_cost(1.9 * INSN_COST);
11250   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
11251 
11252   ins_encode %{
11253     __ sub(as_Register($dst$$reg),
11254               as_Register($src1$$reg),
11255               as_Register($src2$$reg),
11256               Assembler::LSR,
11257               $src3$$constant & 0x3f);
11258   %}
11259 
11260   ins_pipe(ialu_reg_reg_shift);
11261 %}
11262 
11263 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11264                          iRegIorL2I src1, iRegIorL2I src2,
11265                          immI src3, rFlagsReg cr) %{
11266   match(Set dst (SubI src1 (RShiftI src2 src3)));
11267 
11268   ins_cost(1.9 * INSN_COST);
11269   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
11270 
11271   ins_encode %{
11272     __ subw(as_Register($dst$$reg),
11273               as_Register($src1$$reg),
11274               as_Register($src2$$reg),
11275               Assembler::ASR,
11276               $src3$$constant & 0x1f);
11277   %}
11278 
11279   ins_pipe(ialu_reg_reg_shift);
11280 %}
11281 
11282 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11283                          iRegL src1, iRegL src2,
11284                          immI src3, rFlagsReg cr) %{
11285   match(Set dst (SubL src1 (RShiftL src2 src3)));
11286 
11287   ins_cost(1.9 * INSN_COST);
11288   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
11289 
11290   ins_encode %{
11291     __ sub(as_Register($dst$$reg),
11292               as_Register($src1$$reg),
11293               as_Register($src2$$reg),
11294               Assembler::ASR,
11295               $src3$$constant & 0x3f);
11296   %}
11297 
11298   ins_pipe(ialu_reg_reg_shift);
11299 %}
11300 
11301 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11302                          iRegIorL2I src1, iRegIorL2I src2,
11303                          immI src3, rFlagsReg cr) %{
11304   match(Set dst (SubI src1 (LShiftI src2 src3)));
11305 
11306   ins_cost(1.9 * INSN_COST);
11307   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
11308 
11309   ins_encode %{
11310     __ subw(as_Register($dst$$reg),
11311               as_Register($src1$$reg),
11312               as_Register($src2$$reg),
11313               Assembler::LSL,
11314               $src3$$constant & 0x1f);
11315   %}
11316 
11317   ins_pipe(ialu_reg_reg_shift);
11318 %}
11319 
11320 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11321                          iRegL src1, iRegL src2,
11322                          immI src3, rFlagsReg cr) %{
11323   match(Set dst (SubL src1 (LShiftL src2 src3)));
11324 
11325   ins_cost(1.9 * INSN_COST);
11326   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
11327 
11328   ins_encode %{
11329     __ sub(as_Register($dst$$reg),
11330               as_Register($src1$$reg),
11331               as_Register($src2$$reg),
11332               Assembler::LSL,
11333               $src3$$constant & 0x3f);
11334   %}
11335 
11336   ins_pipe(ialu_reg_reg_shift);
11337 %}
11338 
11339 
11340 
11341 // Shift Left followed by Shift Right.
11342 // This idiom is used by the compiler for the i2b bytecode etc.
11343 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11344 %{
11345   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11346   ins_cost(INSN_COST * 2);
11347   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11348   ins_encode %{
11349     int lshift = $lshift_count$$constant & 63;
11350     int rshift = $rshift_count$$constant & 63;
11351     int s = 63 - lshift;
11352     int r = (rshift - lshift) & 63;
11353     __ sbfm(as_Register($dst$$reg),
11354             as_Register($src$$reg),
11355             r, s);
11356   %}
11357 
11358   ins_pipe(ialu_reg_shift);
11359 %}
11360 
11361 // Shift Left followed by Shift Right.
11362 // This idiom is used by the compiler for the i2b bytecode etc.
11363 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11364 %{
11365   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11366   ins_cost(INSN_COST * 2);
11367   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11368   ins_encode %{
11369     int lshift = $lshift_count$$constant & 31;
11370     int rshift = $rshift_count$$constant & 31;
11371     int s = 31 - lshift;
11372     int r = (rshift - lshift) & 31;
11373     __ sbfmw(as_Register($dst$$reg),
11374             as_Register($src$$reg),
11375             r, s);
11376   %}
11377 
11378   ins_pipe(ialu_reg_shift);
11379 %}
11380 
11381 // Shift Left followed by Shift Right.
11382 // This idiom is used by the compiler for the i2b bytecode etc.
11383 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11384 %{
11385   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11386   ins_cost(INSN_COST * 2);
11387   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11388   ins_encode %{
11389     int lshift = $lshift_count$$constant & 63;
11390     int rshift = $rshift_count$$constant & 63;
11391     int s = 63 - lshift;
11392     int r = (rshift - lshift) & 63;
11393     __ ubfm(as_Register($dst$$reg),
11394             as_Register($src$$reg),
11395             r, s);
11396   %}
11397 
11398   ins_pipe(ialu_reg_shift);
11399 %}
11400 
11401 // Shift Left followed by Shift Right.
11402 // This idiom is used by the compiler for the i2b bytecode etc.
11403 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11404 %{
11405   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11406   ins_cost(INSN_COST * 2);
11407   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11408   ins_encode %{
11409     int lshift = $lshift_count$$constant & 31;
11410     int rshift = $rshift_count$$constant & 31;
11411     int s = 31 - lshift;
11412     int r = (rshift - lshift) & 31;
11413     __ ubfmw(as_Register($dst$$reg),
11414             as_Register($src$$reg),
11415             r, s);
11416   %}
11417 
11418   ins_pipe(ialu_reg_shift);
11419 %}
11420 // Bitfield extract with shift & mask
11421 
11422 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11423 %{
11424   match(Set dst (AndI (URShiftI src rshift) mask));
11425   // Make sure we are not going to exceed what ubfxw can do.
11426   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11427 
11428   ins_cost(INSN_COST);
11429   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11430   ins_encode %{
11431     int rshift = $rshift$$constant & 31;
11432     long mask = $mask$$constant;
11433     int width = exact_log2(mask+1);
11434     __ ubfxw(as_Register($dst$$reg),
11435             as_Register($src$$reg), rshift, width);
11436   %}
11437   ins_pipe(ialu_reg_shift);
11438 %}
11439 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11440 %{
11441   match(Set dst (AndL (URShiftL src rshift) mask));
11442   // Make sure we are not going to exceed what ubfx can do.
11443   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11444 
11445   ins_cost(INSN_COST);
11446   format %{ "ubfx $dst, $src, $rshift, $mask" %}
11447   ins_encode %{
11448     int rshift = $rshift$$constant & 63;
11449     long mask = $mask$$constant;
11450     int width = exact_log2_long(mask+1);
11451     __ ubfx(as_Register($dst$$reg),
11452             as_Register($src$$reg), rshift, width);
11453   %}
11454   ins_pipe(ialu_reg_shift);
11455 %}
11456 
11457 // We can use ubfx when extending an And with a mask when we know mask
11458 // is positive.  We know that because immI_bitmask guarantees it.
11459 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11460 %{
11461   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11462   // Make sure we are not going to exceed what ubfxw can do.
11463   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11464 
11465   ins_cost(INSN_COST * 2);
11466   format %{ "ubfx $dst, $src, $rshift, $mask" %}
11467   ins_encode %{
11468     int rshift = $rshift$$constant & 31;
11469     long mask = $mask$$constant;
11470     int width = exact_log2(mask+1);
11471     __ ubfx(as_Register($dst$$reg),
11472             as_Register($src$$reg), rshift, width);
11473   %}
11474   ins_pipe(ialu_reg_shift);
11475 %}
11476 
11477 // We can use ubfiz when masking by a positive number and then left shifting the result.
11478 // We know that the mask is positive because immI_bitmask guarantees it.
11479 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11480 %{
11481   match(Set dst (LShiftI (AndI src mask) lshift));
11482   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11483 
11484   ins_cost(INSN_COST);
11485   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11486   ins_encode %{
11487     int lshift = $lshift$$constant & 31;
11488     long mask = $mask$$constant;
11489     int width = exact_log2(mask+1);
11490     __ ubfizw(as_Register($dst$$reg),
11491           as_Register($src$$reg), lshift, width);
11492   %}
11493   ins_pipe(ialu_reg_shift);
11494 %}
11495 // We can use ubfiz when masking by a positive number and then left shifting the result.
11496 // We know that the mask is positive because immL_bitmask guarantees it.
11497 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11498 %{
11499   match(Set dst (LShiftL (AndL src mask) lshift));
11500   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11501 
11502   ins_cost(INSN_COST);
11503   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11504   ins_encode %{
11505     int lshift = $lshift$$constant & 63;
11506     long mask = $mask$$constant;
11507     int width = exact_log2_long(mask+1);
11508     __ ubfiz(as_Register($dst$$reg),
11509           as_Register($src$$reg), lshift, width);
11510   %}
11511   ins_pipe(ialu_reg_shift);
11512 %}
11513 
11514 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11515 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11516 %{
11517   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11518   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11519 
11520   ins_cost(INSN_COST);
11521   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11522   ins_encode %{
11523     int lshift = $lshift$$constant & 63;
11524     long mask = $mask$$constant;
11525     int width = exact_log2(mask+1);
11526     __ ubfiz(as_Register($dst$$reg),
11527              as_Register($src$$reg), lshift, width);
11528   %}
11529   ins_pipe(ialu_reg_shift);
11530 %}
11531 
11532 // Rotations
11533 
11534 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11535 %{
11536   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11537   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11538 
11539   ins_cost(INSN_COST);
11540   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11541 
11542   ins_encode %{
11543     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11544             $rshift$$constant & 63);
11545   %}
11546   ins_pipe(ialu_reg_reg_extr);
11547 %}
11548 
11549 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11550 %{
11551   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11552   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11553 
11554   ins_cost(INSN_COST);
11555   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11556 
11557   ins_encode %{
11558     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11559             $rshift$$constant & 31);
11560   %}
11561   ins_pipe(ialu_reg_reg_extr);
11562 %}
11563 
11564 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11565 %{
11566   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11567   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11568 
11569   ins_cost(INSN_COST);
11570   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11571 
11572   ins_encode %{
11573     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11574             $rshift$$constant & 63);
11575   %}
11576   ins_pipe(ialu_reg_reg_extr);
11577 %}
11578 
11579 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11580 %{
11581   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11582   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11583 
11584   ins_cost(INSN_COST);
11585   format %{ "extr $dst, $src1, $src2, #$rshift" %}
11586 
11587   ins_encode %{
11588     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11589             $rshift$$constant & 31);
11590   %}
11591   ins_pipe(ialu_reg_reg_extr);
11592 %}
11593 
11594 
11595 // rol expander
11596 
11597 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11598 %{
11599   effect(DEF dst, USE src, USE shift);
11600 
11601   format %{ "rol    $dst, $src, $shift" %}
11602   ins_cost(INSN_COST * 3);
11603   ins_encode %{
11604     __ subw(rscratch1, zr, as_Register($shift$$reg));
11605     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11606             rscratch1);
11607     %}
11608   ins_pipe(ialu_reg_reg_vshift);
11609 %}
11610 
11611 // rol expander
11612 
11613 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11614 %{
11615   effect(DEF dst, USE src, USE shift);
11616 
11617   format %{ "rol    $dst, $src, $shift" %}
11618   ins_cost(INSN_COST * 3);
11619   ins_encode %{
11620     __ subw(rscratch1, zr, as_Register($shift$$reg));
11621     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11622             rscratch1);
11623     %}
11624   ins_pipe(ialu_reg_reg_vshift);
11625 %}
11626 
11627 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11628 %{
11629   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
11630 
11631   expand %{
11632     rolL_rReg(dst, src, shift, cr);
11633   %}
11634 %}
11635 
11636 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11637 %{
11638   match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
11639 
11640   expand %{
11641     rolL_rReg(dst, src, shift, cr);
11642   %}
11643 %}
11644 
11645 instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11646 %{
11647   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
11648 
11649   expand %{
11650     rolI_rReg(dst, src, shift, cr);
11651   %}
11652 %}
11653 
11654 instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11655 %{
11656   match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
11657 
11658   expand %{
11659     rolI_rReg(dst, src, shift, cr);
11660   %}
11661 %}
11662 
11663 // ror expander
11664 
11665 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11666 %{
11667   effect(DEF dst, USE src, USE shift);
11668 
11669   format %{ "ror    $dst, $src, $shift" %}
11670   ins_cost(INSN_COST);
11671   ins_encode %{
11672     __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11673             as_Register($shift$$reg));
11674     %}
11675   ins_pipe(ialu_reg_reg_vshift);
11676 %}
11677 
11678 // ror expander
11679 
11680 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11681 %{
11682   effect(DEF dst, USE src, USE shift);
11683 
11684   format %{ "ror    $dst, $src, $shift" %}
11685   ins_cost(INSN_COST);
11686   ins_encode %{
11687     __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11688             as_Register($shift$$reg));
11689     %}
11690   ins_pipe(ialu_reg_reg_vshift);
11691 %}
11692 
11693 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11694 %{
11695   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
11696 
11697   expand %{
11698     rorL_rReg(dst, src, shift, cr);
11699   %}
11700 %}
11701 
11702 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11703 %{
11704   match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
11705 
11706   expand %{
11707     rorL_rReg(dst, src, shift, cr);
11708   %}
11709 %}
11710 
11711 instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11712 %{
11713   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
11714 
11715   expand %{
11716     rorI_rReg(dst, src, shift, cr);
11717   %}
11718 %}
11719 
11720 instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11721 %{
11722   match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
11723 
11724   expand %{
11725     rorI_rReg(dst, src, shift, cr);
11726   %}
11727 %}
11728 
11729 // Add/subtract (extended)
11730 
11731 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11732 %{
11733   match(Set dst (AddL src1 (ConvI2L src2)));
11734   ins_cost(INSN_COST);
11735   format %{ "add  $dst, $src1, $src2, sxtw" %}
11736 
11737    ins_encode %{
11738      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11739             as_Register($src2$$reg), ext::sxtw);
11740    %}
11741   ins_pipe(ialu_reg_reg);
11742 %};
11743 
11744 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11745 %{
11746   match(Set dst (SubL src1 (ConvI2L src2)));
11747   ins_cost(INSN_COST);
11748   format %{ "sub  $dst, $src1, $src2, sxtw" %}
11749 
11750    ins_encode %{
11751      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11752             as_Register($src2$$reg), ext::sxtw);
11753    %}
11754   ins_pipe(ialu_reg_reg);
11755 %};
11756 
11757 
11758 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11759 %{
11760   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11761   ins_cost(INSN_COST);
11762   format %{ "add  $dst, $src1, $src2, sxth" %}
11763 
11764    ins_encode %{
11765      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11766             as_Register($src2$$reg), ext::sxth);
11767    %}
11768   ins_pipe(ialu_reg_reg);
11769 %}
11770 
11771 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11772 %{
11773   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11774   ins_cost(INSN_COST);
11775   format %{ "add  $dst, $src1, $src2, sxtb" %}
11776 
11777    ins_encode %{
11778      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11779             as_Register($src2$$reg), ext::sxtb);
11780    %}
11781   ins_pipe(ialu_reg_reg);
11782 %}
11783 
11784 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11785 %{
11786   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11787   ins_cost(INSN_COST);
11788   format %{ "add  $dst, $src1, $src2, uxtb" %}
11789 
11790    ins_encode %{
11791      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11792             as_Register($src2$$reg), ext::uxtb);
11793    %}
11794   ins_pipe(ialu_reg_reg);
11795 %}
11796 
11797 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11798 %{
11799   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11800   ins_cost(INSN_COST);
11801   format %{ "add  $dst, $src1, $src2, sxth" %}
11802 
11803    ins_encode %{
11804      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11805             as_Register($src2$$reg), ext::sxth);
11806    %}
11807   ins_pipe(ialu_reg_reg);
11808 %}
11809 
11810 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11811 %{
11812   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11813   ins_cost(INSN_COST);
11814   format %{ "add  $dst, $src1, $src2, sxtw" %}
11815 
11816    ins_encode %{
11817      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11818             as_Register($src2$$reg), ext::sxtw);
11819    %}
11820   ins_pipe(ialu_reg_reg);
11821 %}
11822 
11823 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11824 %{
11825   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11826   ins_cost(INSN_COST);
11827   format %{ "add  $dst, $src1, $src2, sxtb" %}
11828 
11829    ins_encode %{
11830      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11831             as_Register($src2$$reg), ext::sxtb);
11832    %}
11833   ins_pipe(ialu_reg_reg);
11834 %}
11835 
11836 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11837 %{
11838   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11839   ins_cost(INSN_COST);
11840   format %{ "add  $dst, $src1, $src2, uxtb" %}
11841 
11842    ins_encode %{
11843      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11844             as_Register($src2$$reg), ext::uxtb);
11845    %}
11846   ins_pipe(ialu_reg_reg);
11847 %}
11848 
11849 
11850 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11851 %{
11852   match(Set dst (AddI src1 (AndI src2 mask)));
11853   ins_cost(INSN_COST);
11854   format %{ "addw  $dst, $src1, $src2, uxtb" %}
11855 
11856    ins_encode %{
11857      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11858             as_Register($src2$$reg), ext::uxtb);
11859    %}
11860   ins_pipe(ialu_reg_reg);
11861 %}
11862 
11863 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11864 %{
11865   match(Set dst (AddI src1 (AndI src2 mask)));
11866   ins_cost(INSN_COST);
11867   format %{ "addw  $dst, $src1, $src2, uxth" %}
11868 
11869    ins_encode %{
11870      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11871             as_Register($src2$$reg), ext::uxth);
11872    %}
11873   ins_pipe(ialu_reg_reg);
11874 %}
11875 
11876 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11877 %{
11878   match(Set dst (AddL src1 (AndL src2 mask)));
11879   ins_cost(INSN_COST);
11880   format %{ "add  $dst, $src1, $src2, uxtb" %}
11881 
11882    ins_encode %{
11883      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11884             as_Register($src2$$reg), ext::uxtb);
11885    %}
11886   ins_pipe(ialu_reg_reg);
11887 %}
11888 
11889 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11890 %{
11891   match(Set dst (AddL src1 (AndL src2 mask)));
11892   ins_cost(INSN_COST);
11893   format %{ "add  $dst, $src1, $src2, uxth" %}
11894 
11895    ins_encode %{
11896      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11897             as_Register($src2$$reg), ext::uxth);
11898    %}
11899   ins_pipe(ialu_reg_reg);
11900 %}
11901 
11902 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
11903 %{
11904   match(Set dst (AddL src1 (AndL src2 mask)));
11905   ins_cost(INSN_COST);
11906   format %{ "add  $dst, $src1, $src2, uxtw" %}
11907 
11908    ins_encode %{
11909      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11910             as_Register($src2$$reg), ext::uxtw);
11911    %}
11912   ins_pipe(ialu_reg_reg);
11913 %}
11914 
11915 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11916 %{
11917   match(Set dst (SubI src1 (AndI src2 mask)));
11918   ins_cost(INSN_COST);
11919   format %{ "subw  $dst, $src1, $src2, uxtb" %}
11920 
11921    ins_encode %{
11922      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11923             as_Register($src2$$reg), ext::uxtb);
11924    %}
11925   ins_pipe(ialu_reg_reg);
11926 %}
11927 
11928 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11929 %{
11930   match(Set dst (SubI src1 (AndI src2 mask)));
11931   ins_cost(INSN_COST);
11932   format %{ "subw  $dst, $src1, $src2, uxth" %}
11933 
11934    ins_encode %{
11935      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11936             as_Register($src2$$reg), ext::uxth);
11937    %}
11938   ins_pipe(ialu_reg_reg);
11939 %}
11940 
11941 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11942 %{
11943   match(Set dst (SubL src1 (AndL src2 mask)));
11944   ins_cost(INSN_COST);
11945   format %{ "sub  $dst, $src1, $src2, uxtb" %}
11946 
11947    ins_encode %{
11948      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11949             as_Register($src2$$reg), ext::uxtb);
11950    %}
11951   ins_pipe(ialu_reg_reg);
11952 %}
11953 
11954 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11955 %{
11956   match(Set dst (SubL src1 (AndL src2 mask)));
11957   ins_cost(INSN_COST);
11958   format %{ "sub  $dst, $src1, $src2, uxth" %}
11959 
11960    ins_encode %{
11961      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11962             as_Register($src2$$reg), ext::uxth);
11963    %}
11964   ins_pipe(ialu_reg_reg);
11965 %}
11966 
11967 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
11968 %{
11969   match(Set dst (SubL src1 (AndL src2 mask)));
11970   ins_cost(INSN_COST);
11971   format %{ "sub  $dst, $src1, $src2, uxtw" %}
11972 
11973    ins_encode %{
11974      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11975             as_Register($src2$$reg), ext::uxtw);
11976    %}
11977   ins_pipe(ialu_reg_reg);
11978 %}
11979 
11980 
11981 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
11982 %{
11983   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
11984   ins_cost(1.9 * INSN_COST);
11985   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
11986 
11987    ins_encode %{
11988      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11989             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
11990    %}
11991   ins_pipe(ialu_reg_reg_shift);
11992 %}
11993 
11994 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
11995 %{
11996   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
11997   ins_cost(1.9 * INSN_COST);
11998   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
11999 
12000    ins_encode %{
12001      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12002             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12003    %}
12004   ins_pipe(ialu_reg_reg_shift);
12005 %}
12006 
12007 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12008 %{
12009   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12010   ins_cost(1.9 * INSN_COST);
12011   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
12012 
12013    ins_encode %{
12014      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12015             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12016    %}
12017   ins_pipe(ialu_reg_reg_shift);
12018 %}
12019 
12020 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12021 %{
12022   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12023   ins_cost(1.9 * INSN_COST);
12024   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
12025 
12026    ins_encode %{
12027      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12028             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12029    %}
12030   ins_pipe(ialu_reg_reg_shift);
12031 %}
12032 
12033 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12034 %{
12035   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12036   ins_cost(1.9 * INSN_COST);
12037   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
12038 
12039    ins_encode %{
12040      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12041             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12042    %}
12043   ins_pipe(ialu_reg_reg_shift);
12044 %}
12045 
12046 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12047 %{
12048   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12049   ins_cost(1.9 * INSN_COST);
12050   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
12051 
12052    ins_encode %{
12053      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12054             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12055    %}
12056   ins_pipe(ialu_reg_reg_shift);
12057 %}
12058 
12059 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12060 %{
12061   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12062   ins_cost(1.9 * INSN_COST);
12063   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
12064 
12065    ins_encode %{
12066      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12067             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12068    %}
12069   ins_pipe(ialu_reg_reg_shift);
12070 %}
12071 
12072 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12073 %{
12074   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12075   ins_cost(1.9 * INSN_COST);
12076   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
12077 
12078    ins_encode %{
12079      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12080             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12081    %}
12082   ins_pipe(ialu_reg_reg_shift);
12083 %}
12084 
12085 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12086 %{
12087   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12088   ins_cost(1.9 * INSN_COST);
12089   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
12090 
12091    ins_encode %{
12092      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12093             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12094    %}
12095   ins_pipe(ialu_reg_reg_shift);
12096 %}
12097 
12098 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12099 %{
12100   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12101   ins_cost(1.9 * INSN_COST);
12102   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
12103 
12104    ins_encode %{
12105      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12106             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12107    %}
12108   ins_pipe(ialu_reg_reg_shift);
12109 %}
12110 
12111 
12112 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12113 %{
12114   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12115   ins_cost(1.9 * INSN_COST);
12116   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
12117 
12118    ins_encode %{
12119      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12120             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12121    %}
12122   ins_pipe(ialu_reg_reg_shift);
12123 %};
12124 
12125 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12126 %{
12127   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12128   ins_cost(1.9 * INSN_COST);
12129   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
12130 
12131    ins_encode %{
12132      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12133             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12134    %}
12135   ins_pipe(ialu_reg_reg_shift);
12136 %};
12137 
12138 
12139 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12140 %{
12141   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12142   ins_cost(1.9 * INSN_COST);
12143   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
12144 
12145    ins_encode %{
12146      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12147             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12148    %}
12149   ins_pipe(ialu_reg_reg_shift);
12150 %}
12151 
12152 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12153 %{
12154   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12155   ins_cost(1.9 * INSN_COST);
12156   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
12157 
12158    ins_encode %{
12159      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12160             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12161    %}
12162   ins_pipe(ialu_reg_reg_shift);
12163 %}
12164 
12165 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12166 %{
12167   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12168   ins_cost(1.9 * INSN_COST);
12169   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
12170 
12171    ins_encode %{
12172      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12173             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12174    %}
12175   ins_pipe(ialu_reg_reg_shift);
12176 %}
12177 
12178 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12179 %{
12180   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12181   ins_cost(1.9 * INSN_COST);
12182   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
12183 
12184    ins_encode %{
12185      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12186             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12187    %}
12188   ins_pipe(ialu_reg_reg_shift);
12189 %}
12190 
12191 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12192 %{
12193   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12194   ins_cost(1.9 * INSN_COST);
12195   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
12196 
12197    ins_encode %{
12198      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12199             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12200    %}
12201   ins_pipe(ialu_reg_reg_shift);
12202 %}
12203 
12204 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12205 %{
12206   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12207   ins_cost(1.9 * INSN_COST);
12208   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
12209 
12210    ins_encode %{
12211      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12212             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12213    %}
12214   ins_pipe(ialu_reg_reg_shift);
12215 %}
12216 
12217 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12218 %{
12219   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12220   ins_cost(1.9 * INSN_COST);
12221   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
12222 
12223    ins_encode %{
12224      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12225             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12226    %}
12227   ins_pipe(ialu_reg_reg_shift);
12228 %}
12229 
12230 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12231 %{
12232   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12233   ins_cost(1.9 * INSN_COST);
12234   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
12235 
12236    ins_encode %{
12237      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12238             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12239    %}
12240   ins_pipe(ialu_reg_reg_shift);
12241 %}
12242 
12243 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12244 %{
12245   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12246   ins_cost(1.9 * INSN_COST);
12247   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
12248 
12249    ins_encode %{
12250      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12251             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12252    %}
12253   ins_pipe(ialu_reg_reg_shift);
12254 %}
12255 
12256 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12257 %{
12258   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12259   ins_cost(1.9 * INSN_COST);
12260   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
12261 
12262    ins_encode %{
12263      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12264             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12265    %}
12266   ins_pipe(ialu_reg_reg_shift);
12267 %}
12268 // END This section of the file is automatically generated. Do not edit --------------
12269 
12270 // ============================================================================
12271 // Floating Point Arithmetic Instructions
12272 
12273 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12274   match(Set dst (AddF src1 src2));
12275 
12276   ins_cost(INSN_COST * 5);
12277   format %{ "fadds   $dst, $src1, $src2" %}
12278 
12279   ins_encode %{
12280     __ fadds(as_FloatRegister($dst$$reg),
12281              as_FloatRegister($src1$$reg),
12282              as_FloatRegister($src2$$reg));
12283   %}
12284 
12285   ins_pipe(fp_dop_reg_reg_s);
12286 %}
12287 
12288 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12289   match(Set dst (AddD src1 src2));
12290 
12291   ins_cost(INSN_COST * 5);
12292   format %{ "faddd   $dst, $src1, $src2" %}
12293 
12294   ins_encode %{
12295     __ faddd(as_FloatRegister($dst$$reg),
12296              as_FloatRegister($src1$$reg),
12297              as_FloatRegister($src2$$reg));
12298   %}
12299 
12300   ins_pipe(fp_dop_reg_reg_d);
12301 %}
12302 
12303 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12304   match(Set dst (SubF src1 src2));
12305 
12306   ins_cost(INSN_COST * 5);
12307   format %{ "fsubs   $dst, $src1, $src2" %}
12308 
12309   ins_encode %{
12310     __ fsubs(as_FloatRegister($dst$$reg),
12311              as_FloatRegister($src1$$reg),
12312              as_FloatRegister($src2$$reg));
12313   %}
12314 
12315   ins_pipe(fp_dop_reg_reg_s);
12316 %}
12317 
12318 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12319   match(Set dst (SubD src1 src2));
12320 
12321   ins_cost(INSN_COST * 5);
12322   format %{ "fsubd   $dst, $src1, $src2" %}
12323 
12324   ins_encode %{
12325     __ fsubd(as_FloatRegister($dst$$reg),
12326              as_FloatRegister($src1$$reg),
12327              as_FloatRegister($src2$$reg));
12328   %}
12329 
12330   ins_pipe(fp_dop_reg_reg_d);
12331 %}
12332 
12333 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12334   match(Set dst (MulF src1 src2));
12335 
12336   ins_cost(INSN_COST * 6);
12337   format %{ "fmuls   $dst, $src1, $src2" %}
12338 
12339   ins_encode %{
12340     __ fmuls(as_FloatRegister($dst$$reg),
12341              as_FloatRegister($src1$$reg),
12342              as_FloatRegister($src2$$reg));
12343   %}
12344 
12345   ins_pipe(fp_dop_reg_reg_s);
12346 %}
12347 
12348 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12349   match(Set dst (MulD src1 src2));
12350 
12351   ins_cost(INSN_COST * 6);
12352   format %{ "fmuld   $dst, $src1, $src2" %}
12353 
12354   ins_encode %{
12355     __ fmuld(as_FloatRegister($dst$$reg),
12356              as_FloatRegister($src1$$reg),
12357              as_FloatRegister($src2$$reg));
12358   %}
12359 
12360   ins_pipe(fp_dop_reg_reg_d);
12361 %}
12362 
12363 // src1 * src2 + src3
12364 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12365   predicate(UseFMA);
12366   match(Set dst (FmaF src3 (Binary src1 src2)));
12367 
12368   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
12369 
12370   ins_encode %{
12371     __ fmadds(as_FloatRegister($dst$$reg),
12372              as_FloatRegister($src1$$reg),
12373              as_FloatRegister($src2$$reg),
12374              as_FloatRegister($src3$$reg));
12375   %}
12376 
12377   ins_pipe(pipe_class_default);
12378 %}
12379 
12380 // src1 * src2 + src3
12381 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12382   predicate(UseFMA);
12383   match(Set dst (FmaD src3 (Binary src1 src2)));
12384 
12385   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
12386 
12387   ins_encode %{
12388     __ fmaddd(as_FloatRegister($dst$$reg),
12389              as_FloatRegister($src1$$reg),
12390              as_FloatRegister($src2$$reg),
12391              as_FloatRegister($src3$$reg));
12392   %}
12393 
12394   ins_pipe(pipe_class_default);
12395 %}
12396 
12397 // -src1 * src2 + src3
12398 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12399   predicate(UseFMA);
12400   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
12401   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12402 
12403   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
12404 
12405   ins_encode %{
12406     __ fmsubs(as_FloatRegister($dst$$reg),
12407               as_FloatRegister($src1$$reg),
12408               as_FloatRegister($src2$$reg),
12409               as_FloatRegister($src3$$reg));
12410   %}
12411 
12412   ins_pipe(pipe_class_default);
12413 %}
12414 
12415 // -src1 * src2 + src3
12416 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12417   predicate(UseFMA);
12418   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
12419   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12420 
12421   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
12422 
12423   ins_encode %{
12424     __ fmsubd(as_FloatRegister($dst$$reg),
12425               as_FloatRegister($src1$$reg),
12426               as_FloatRegister($src2$$reg),
12427               as_FloatRegister($src3$$reg));
12428   %}
12429 
12430   ins_pipe(pipe_class_default);
12431 %}
12432 
12433 // -src1 * src2 - src3
12434 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12435   predicate(UseFMA);
12436   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
12437   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12438 
12439   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
12440 
12441   ins_encode %{
12442     __ fnmadds(as_FloatRegister($dst$$reg),
12443                as_FloatRegister($src1$$reg),
12444                as_FloatRegister($src2$$reg),
12445                as_FloatRegister($src3$$reg));
12446   %}
12447 
12448   ins_pipe(pipe_class_default);
12449 %}
12450 
12451 // -src1 * src2 - src3
12452 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12453   predicate(UseFMA);
12454   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
12455   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
12456 
12457   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
12458 
12459   ins_encode %{
12460     __ fnmaddd(as_FloatRegister($dst$$reg),
12461                as_FloatRegister($src1$$reg),
12462                as_FloatRegister($src2$$reg),
12463                as_FloatRegister($src3$$reg));
12464   %}
12465 
12466   ins_pipe(pipe_class_default);
12467 %}
12468 
12469 // src1 * src2 - src3
12470 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
12471   predicate(UseFMA);
12472   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
12473 
12474   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
12475 
12476   ins_encode %{
12477     __ fnmsubs(as_FloatRegister($dst$$reg),
12478                as_FloatRegister($src1$$reg),
12479                as_FloatRegister($src2$$reg),
12480                as_FloatRegister($src3$$reg));
12481   %}
12482 
12483   ins_pipe(pipe_class_default);
12484 %}
12485 
12486 // src1 * src2 - src3
12487 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
12488   predicate(UseFMA);
12489   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
12490 
12491   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
12492 
12493   ins_encode %{
12494   // n.b. insn name should be fnmsubd
12495     __ fnmsub(as_FloatRegister($dst$$reg),
12496               as_FloatRegister($src1$$reg),
12497               as_FloatRegister($src2$$reg),
12498               as_FloatRegister($src3$$reg));
12499   %}
12500 
12501   ins_pipe(pipe_class_default);
12502 %}
12503 
12504 
12505 // Math.max(FF)F
12506 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12507   match(Set dst (MaxF src1 src2));
12508 
12509   format %{ "fmaxs   $dst, $src1, $src2" %}
12510   ins_encode %{
12511     __ fmaxs(as_FloatRegister($dst$$reg),
12512              as_FloatRegister($src1$$reg),
12513              as_FloatRegister($src2$$reg));
12514   %}
12515 
12516   ins_pipe(fp_dop_reg_reg_s);
12517 %}
12518 
12519 // Math.min(FF)F
12520 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12521   match(Set dst (MinF src1 src2));
12522 
12523   format %{ "fmins   $dst, $src1, $src2" %}
12524   ins_encode %{
12525     __ fmins(as_FloatRegister($dst$$reg),
12526              as_FloatRegister($src1$$reg),
12527              as_FloatRegister($src2$$reg));
12528   %}
12529 
12530   ins_pipe(fp_dop_reg_reg_s);
12531 %}
12532 
12533 // Math.max(DD)D
12534 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12535   match(Set dst (MaxD src1 src2));
12536 
12537   format %{ "fmaxd   $dst, $src1, $src2" %}
12538   ins_encode %{
12539     __ fmaxd(as_FloatRegister($dst$$reg),
12540              as_FloatRegister($src1$$reg),
12541              as_FloatRegister($src2$$reg));
12542   %}
12543 
12544   ins_pipe(fp_dop_reg_reg_d);
12545 %}
12546 
12547 // Math.min(DD)D
12548 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12549   match(Set dst (MinD src1 src2));
12550 
12551   format %{ "fmind   $dst, $src1, $src2" %}
12552   ins_encode %{
12553     __ fmind(as_FloatRegister($dst$$reg),
12554              as_FloatRegister($src1$$reg),
12555              as_FloatRegister($src2$$reg));
12556   %}
12557 
12558   ins_pipe(fp_dop_reg_reg_d);
12559 %}
12560 
12561 
12562 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12563   match(Set dst (DivF src1  src2));
12564 
12565   ins_cost(INSN_COST * 18);
12566   format %{ "fdivs   $dst, $src1, $src2" %}
12567 
12568   ins_encode %{
12569     __ fdivs(as_FloatRegister($dst$$reg),
12570              as_FloatRegister($src1$$reg),
12571              as_FloatRegister($src2$$reg));
12572   %}
12573 
12574   ins_pipe(fp_div_s);
12575 %}
12576 
12577 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12578   match(Set dst (DivD src1  src2));
12579 
12580   ins_cost(INSN_COST * 32);
12581   format %{ "fdivd   $dst, $src1, $src2" %}
12582 
12583   ins_encode %{
12584     __ fdivd(as_FloatRegister($dst$$reg),
12585              as_FloatRegister($src1$$reg),
12586              as_FloatRegister($src2$$reg));
12587   %}
12588 
12589   ins_pipe(fp_div_d);
12590 %}
12591 
12592 instruct negF_reg_reg(vRegF dst, vRegF src) %{
12593   match(Set dst (NegF src));
12594 
12595   ins_cost(INSN_COST * 3);
12596   format %{ "fneg   $dst, $src" %}
12597 
12598   ins_encode %{
12599     __ fnegs(as_FloatRegister($dst$$reg),
12600              as_FloatRegister($src$$reg));
12601   %}
12602 
12603   ins_pipe(fp_uop_s);
12604 %}
12605 
12606 instruct negD_reg_reg(vRegD dst, vRegD src) %{
12607   match(Set dst (NegD src));
12608 
12609   ins_cost(INSN_COST * 3);
12610   format %{ "fnegd   $dst, $src" %}
12611 
12612   ins_encode %{
12613     __ fnegd(as_FloatRegister($dst$$reg),
12614              as_FloatRegister($src$$reg));
12615   %}
12616 
12617   ins_pipe(fp_uop_d);
12618 %}
12619 
12620 instruct absF_reg(vRegF dst, vRegF src) %{
12621   match(Set dst (AbsF src));
12622 
12623   ins_cost(INSN_COST * 3);
12624   format %{ "fabss   $dst, $src" %}
12625   ins_encode %{
12626     __ fabss(as_FloatRegister($dst$$reg),
12627              as_FloatRegister($src$$reg));
12628   %}
12629 
12630   ins_pipe(fp_uop_s);
12631 %}
12632 
12633 instruct absD_reg(vRegD dst, vRegD src) %{
12634   match(Set dst (AbsD src));
12635 
12636   ins_cost(INSN_COST * 3);
12637   format %{ "fabsd   $dst, $src" %}
12638   ins_encode %{
12639     __ fabsd(as_FloatRegister($dst$$reg),
12640              as_FloatRegister($src$$reg));
12641   %}
12642 
12643   ins_pipe(fp_uop_d);
12644 %}
12645 
12646 instruct sqrtD_reg(vRegD dst, vRegD src) %{
12647   match(Set dst (SqrtD src));
12648 
12649   ins_cost(INSN_COST * 50);
12650   format %{ "fsqrtd  $dst, $src" %}
12651   ins_encode %{
12652     __ fsqrtd(as_FloatRegister($dst$$reg),
12653              as_FloatRegister($src$$reg));
12654   %}
12655 
12656   ins_pipe(fp_div_s);
12657 %}
12658 
12659 instruct sqrtF_reg(vRegF dst, vRegF src) %{
12660   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
12661 
12662   ins_cost(INSN_COST * 50);
12663   format %{ "fsqrts  $dst, $src" %}
12664   ins_encode %{
12665     __ fsqrts(as_FloatRegister($dst$$reg),
12666              as_FloatRegister($src$$reg));
12667   %}
12668 
12669   ins_pipe(fp_div_d);
12670 %}
12671 
12672 // ============================================================================
12673 // Logical Instructions
12674 
12675 // Integer Logical Instructions
12676 
12677 // And Instructions
12678 
12679 
12680 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
12681   match(Set dst (AndI src1 src2));
12682 
12683   format %{ "andw  $dst, $src1, $src2\t# int" %}
12684 
12685   ins_cost(INSN_COST);
12686   ins_encode %{
12687     __ andw(as_Register($dst$$reg),
12688             as_Register($src1$$reg),
12689             as_Register($src2$$reg));
12690   %}
12691 
12692   ins_pipe(ialu_reg_reg);
12693 %}
12694 
12695 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
12696   match(Set dst (AndI src1 src2));
12697 
12698   format %{ "andsw  $dst, $src1, $src2\t# int" %}
12699 
12700   ins_cost(INSN_COST);
12701   ins_encode %{
12702     __ andw(as_Register($dst$$reg),
12703             as_Register($src1$$reg),
12704             (unsigned long)($src2$$constant));
12705   %}
12706 
12707   ins_pipe(ialu_reg_imm);
12708 %}
12709 
12710 // Or Instructions
12711 
12712 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
12713   match(Set dst (OrI src1 src2));
12714 
12715   format %{ "orrw  $dst, $src1, $src2\t# int" %}
12716 
12717   ins_cost(INSN_COST);
12718   ins_encode %{
12719     __ orrw(as_Register($dst$$reg),
12720             as_Register($src1$$reg),
12721             as_Register($src2$$reg));
12722   %}
12723 
12724   ins_pipe(ialu_reg_reg);
12725 %}
12726 
12727 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
12728   match(Set dst (OrI src1 src2));
12729 
12730   format %{ "orrw  $dst, $src1, $src2\t# int" %}
12731 
12732   ins_cost(INSN_COST);
12733   ins_encode %{
12734     __ orrw(as_Register($dst$$reg),
12735             as_Register($src1$$reg),
12736             (unsigned long)($src2$$constant));
12737   %}
12738 
12739   ins_pipe(ialu_reg_imm);
12740 %}
12741 
12742 // Xor Instructions
12743 
12744 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
12745   match(Set dst (XorI src1 src2));
12746 
12747   format %{ "eorw  $dst, $src1, $src2\t# int" %}
12748 
12749   ins_cost(INSN_COST);
12750   ins_encode %{
12751     __ eorw(as_Register($dst$$reg),
12752             as_Register($src1$$reg),
12753             as_Register($src2$$reg));
12754   %}
12755 
12756   ins_pipe(ialu_reg_reg);
12757 %}
12758 
12759 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
12760   match(Set dst (XorI src1 src2));
12761 
12762   format %{ "eorw  $dst, $src1, $src2\t# int" %}
12763 
12764   ins_cost(INSN_COST);
12765   ins_encode %{
12766     __ eorw(as_Register($dst$$reg),
12767             as_Register($src1$$reg),
12768             (unsigned long)($src2$$constant));
12769   %}
12770 
12771   ins_pipe(ialu_reg_imm);
12772 %}
12773 
12774 // Long Logical Instructions
12775 // TODO
12776 
12777 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
12778   match(Set dst (AndL src1 src2));
12779 
12780   format %{ "and  $dst, $src1, $src2\t# int" %}
12781 
12782   ins_cost(INSN_COST);
12783   ins_encode %{
12784     __ andr(as_Register($dst$$reg),
12785             as_Register($src1$$reg),
12786             as_Register($src2$$reg));
12787   %}
12788 
12789   ins_pipe(ialu_reg_reg);
12790 %}
12791 
12792 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
12793   match(Set dst (AndL src1 src2));
12794 
12795   format %{ "and  $dst, $src1, $src2\t# int" %}
12796 
12797   ins_cost(INSN_COST);
12798   ins_encode %{
12799     __ andr(as_Register($dst$$reg),
12800             as_Register($src1$$reg),
12801             (unsigned long)($src2$$constant));
12802   %}
12803 
12804   ins_pipe(ialu_reg_imm);
12805 %}
12806 
12807 // Or Instructions
12808 
12809 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
12810   match(Set dst (OrL src1 src2));
12811 
12812   format %{ "orr  $dst, $src1, $src2\t# int" %}
12813 
12814   ins_cost(INSN_COST);
12815   ins_encode %{
12816     __ orr(as_Register($dst$$reg),
12817            as_Register($src1$$reg),
12818            as_Register($src2$$reg));
12819   %}
12820 
12821   ins_pipe(ialu_reg_reg);
12822 %}
12823 
12824 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
12825   match(Set dst (OrL src1 src2));
12826 
12827   format %{ "orr  $dst, $src1, $src2\t# int" %}
12828 
12829   ins_cost(INSN_COST);
12830   ins_encode %{
12831     __ orr(as_Register($dst$$reg),
12832            as_Register($src1$$reg),
12833            (unsigned long)($src2$$constant));
12834   %}
12835 
12836   ins_pipe(ialu_reg_imm);
12837 %}
12838 
12839 // Xor Instructions
12840 
12841 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
12842   match(Set dst (XorL src1 src2));
12843 
12844   format %{ "eor  $dst, $src1, $src2\t# int" %}
12845 
12846   ins_cost(INSN_COST);
12847   ins_encode %{
12848     __ eor(as_Register($dst$$reg),
12849            as_Register($src1$$reg),
12850            as_Register($src2$$reg));
12851   %}
12852 
12853   ins_pipe(ialu_reg_reg);
12854 %}
12855 
12856 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
12857   match(Set dst (XorL src1 src2));
12858 
12859   ins_cost(INSN_COST);
12860   format %{ "eor  $dst, $src1, $src2\t# int" %}
12861 
12862   ins_encode %{
12863     __ eor(as_Register($dst$$reg),
12864            as_Register($src1$$reg),
12865            (unsigned long)($src2$$constant));
12866   %}
12867 
12868   ins_pipe(ialu_reg_imm);
12869 %}
12870 
12871 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
12872 %{
12873   match(Set dst (ConvI2L src));
12874 
12875   ins_cost(INSN_COST);
12876   format %{ "sxtw  $dst, $src\t# i2l" %}
12877   ins_encode %{
12878     __ sbfm($dst$$Register, $src$$Register, 0, 31);
12879   %}
12880   ins_pipe(ialu_reg_shift);
12881 %}
12882 
12883 // this pattern occurs in bigmath arithmetic
12884 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
12885 %{
12886   match(Set dst (AndL (ConvI2L src) mask));
12887 
12888   ins_cost(INSN_COST);
12889   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
12890   ins_encode %{
12891     __ ubfm($dst$$Register, $src$$Register, 0, 31);
12892   %}
12893 
12894   ins_pipe(ialu_reg_shift);
12895 %}
12896 
12897 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
12898   match(Set dst (ConvL2I src));
12899 
12900   ins_cost(INSN_COST);
12901   format %{ "movw  $dst, $src \t// l2i" %}
12902 
12903   ins_encode %{
12904     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
12905   %}
12906 
12907   ins_pipe(ialu_reg);
12908 %}
12909 
12910 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
12911 %{
12912   match(Set dst (Conv2B src));
12913   effect(KILL cr);
12914 
12915   format %{
12916     "cmpw $src, zr\n\t"
12917     "cset $dst, ne"
12918   %}
12919 
12920   ins_encode %{
12921     __ cmpw(as_Register($src$$reg), zr);
12922     __ cset(as_Register($dst$$reg), Assembler::NE);
12923   %}
12924 
12925   ins_pipe(ialu_reg);
12926 %}
12927 
12928 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
12929 %{
12930   match(Set dst (Conv2B src));
12931   effect(KILL cr);
12932 
12933   format %{
12934     "cmp  $src, zr\n\t"
12935     "cset $dst, ne"
12936   %}
12937 
12938   ins_encode %{
12939     __ cmp(as_Register($src$$reg), zr);
12940     __ cset(as_Register($dst$$reg), Assembler::NE);
12941   %}
12942 
12943   ins_pipe(ialu_reg);
12944 %}
12945 
12946 instruct convD2F_reg(vRegF dst, vRegD src) %{
12947   match(Set dst (ConvD2F src));
12948 
12949   ins_cost(INSN_COST * 5);
12950   format %{ "fcvtd  $dst, $src \t// d2f" %}
12951 
12952   ins_encode %{
12953     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
12954   %}
12955 
12956   ins_pipe(fp_d2f);
12957 %}
12958 
12959 instruct convF2D_reg(vRegD dst, vRegF src) %{
12960   match(Set dst (ConvF2D src));
12961 
12962   ins_cost(INSN_COST * 5);
12963   format %{ "fcvts  $dst, $src \t// f2d" %}
12964 
12965   ins_encode %{
12966     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
12967   %}
12968 
12969   ins_pipe(fp_f2d);
12970 %}
12971 
12972 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
12973   match(Set dst (ConvF2I src));
12974 
12975   ins_cost(INSN_COST * 5);
12976   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
12977 
12978   ins_encode %{
12979     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12980   %}
12981 
12982   ins_pipe(fp_f2i);
12983 %}
12984 
12985 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
12986   match(Set dst (ConvF2L src));
12987 
12988   ins_cost(INSN_COST * 5);
12989   format %{ "fcvtzs  $dst, $src \t// f2l" %}
12990 
12991   ins_encode %{
12992     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12993   %}
12994 
12995   ins_pipe(fp_f2l);
12996 %}
12997 
12998 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
12999   match(Set dst (ConvI2F src));
13000 
13001   ins_cost(INSN_COST * 5);
13002   format %{ "scvtfws  $dst, $src \t// i2f" %}
13003 
13004   ins_encode %{
13005     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13006   %}
13007 
13008   ins_pipe(fp_i2f);
13009 %}
13010 
13011 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13012   match(Set dst (ConvL2F src));
13013 
13014   ins_cost(INSN_COST * 5);
13015   format %{ "scvtfs  $dst, $src \t// l2f" %}
13016 
13017   ins_encode %{
13018     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13019   %}
13020 
13021   ins_pipe(fp_l2f);
13022 %}
13023 
13024 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13025   match(Set dst (ConvD2I src));
13026 
13027   ins_cost(INSN_COST * 5);
13028   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
13029 
13030   ins_encode %{
13031     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13032   %}
13033 
13034   ins_pipe(fp_d2i);
13035 %}
13036 
13037 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13038   match(Set dst (ConvD2L src));
13039 
13040   ins_cost(INSN_COST * 5);
13041   format %{ "fcvtzd  $dst, $src \t// d2l" %}
13042 
13043   ins_encode %{
13044     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13045   %}
13046 
13047   ins_pipe(fp_d2l);
13048 %}
13049 
13050 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13051   match(Set dst (ConvI2D src));
13052 
13053   ins_cost(INSN_COST * 5);
13054   format %{ "scvtfwd  $dst, $src \t// i2d" %}
13055 
13056   ins_encode %{
13057     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13058   %}
13059 
13060   ins_pipe(fp_i2d);
13061 %}
13062 
13063 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13064   match(Set dst (ConvL2D src));
13065 
13066   ins_cost(INSN_COST * 5);
13067   format %{ "scvtfd  $dst, $src \t// l2d" %}
13068 
13069   ins_encode %{
13070     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13071   %}
13072 
13073   ins_pipe(fp_l2d);
13074 %}
13075 
13076 // stack <-> reg and reg <-> reg shuffles with no conversion
13077 
13078 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13079 
13080   match(Set dst (MoveF2I src));
13081 
13082   effect(DEF dst, USE src);
13083 
13084   ins_cost(4 * INSN_COST);
13085 
13086   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13087 
13088   ins_encode %{
13089     __ ldrw($dst$$Register, Address(sp, $src$$disp));
13090   %}
13091 
13092   ins_pipe(iload_reg_reg);
13093 
13094 %}
13095 
13096 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13097 
13098   match(Set dst (MoveI2F src));
13099 
13100   effect(DEF dst, USE src);
13101 
13102   ins_cost(4 * INSN_COST);
13103 
13104   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13105 
13106   ins_encode %{
13107     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13108   %}
13109 
13110   ins_pipe(pipe_class_memory);
13111 
13112 %}
13113 
13114 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13115 
13116   match(Set dst (MoveD2L src));
13117 
13118   effect(DEF dst, USE src);
13119 
13120   ins_cost(4 * INSN_COST);
13121 
13122   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13123 
13124   ins_encode %{
13125     __ ldr($dst$$Register, Address(sp, $src$$disp));
13126   %}
13127 
13128   ins_pipe(iload_reg_reg);
13129 
13130 %}
13131 
13132 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13133 
13134   match(Set dst (MoveL2D src));
13135 
13136   effect(DEF dst, USE src);
13137 
13138   ins_cost(4 * INSN_COST);
13139 
13140   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13141 
13142   ins_encode %{
13143     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13144   %}
13145 
13146   ins_pipe(pipe_class_memory);
13147 
13148 %}
13149 
13150 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13151 
13152   match(Set dst (MoveF2I src));
13153 
13154   effect(DEF dst, USE src);
13155 
13156   ins_cost(INSN_COST);
13157 
13158   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13159 
13160   ins_encode %{
13161     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13162   %}
13163 
13164   ins_pipe(pipe_class_memory);
13165 
13166 %}
13167 
13168 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13169 
13170   match(Set dst (MoveI2F src));
13171 
13172   effect(DEF dst, USE src);
13173 
13174   ins_cost(INSN_COST);
13175 
13176   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13177 
13178   ins_encode %{
13179     __ strw($src$$Register, Address(sp, $dst$$disp));
13180   %}
13181 
13182   ins_pipe(istore_reg_reg);
13183 
13184 %}
13185 
13186 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13187 
13188   match(Set dst (MoveD2L src));
13189 
13190   effect(DEF dst, USE src);
13191 
13192   ins_cost(INSN_COST);
13193 
13194   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13195 
13196   ins_encode %{
13197     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13198   %}
13199 
13200   ins_pipe(pipe_class_memory);
13201 
13202 %}
13203 
13204 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13205 
13206   match(Set dst (MoveL2D src));
13207 
13208   effect(DEF dst, USE src);
13209 
13210   ins_cost(INSN_COST);
13211 
13212   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13213 
13214   ins_encode %{
13215     __ str($src$$Register, Address(sp, $dst$$disp));
13216   %}
13217 
13218   ins_pipe(istore_reg_reg);
13219 
13220 %}
13221 
13222 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13223 
13224   match(Set dst (MoveF2I src));
13225 
13226   effect(DEF dst, USE src);
13227 
13228   ins_cost(INSN_COST);
13229 
13230   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
13231 
13232   ins_encode %{
13233     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
13234   %}
13235 
13236   ins_pipe(fp_f2i);
13237 
13238 %}
13239 
13240 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
13241 
13242   match(Set dst (MoveI2F src));
13243 
13244   effect(DEF dst, USE src);
13245 
13246   ins_cost(INSN_COST);
13247 
13248   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
13249 
13250   ins_encode %{
13251     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
13252   %}
13253 
13254   ins_pipe(fp_i2f);
13255 
13256 %}
13257 
13258 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13259 
13260   match(Set dst (MoveD2L src));
13261 
13262   effect(DEF dst, USE src);
13263 
13264   ins_cost(INSN_COST);
13265 
13266   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
13267 
13268   ins_encode %{
13269     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
13270   %}
13271 
13272   ins_pipe(fp_d2l);
13273 
13274 %}
13275 
13276 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
13277 
13278   match(Set dst (MoveL2D src));
13279 
13280   effect(DEF dst, USE src);
13281 
13282   ins_cost(INSN_COST);
13283 
13284   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
13285 
13286   ins_encode %{
13287     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
13288   %}
13289 
13290   ins_pipe(fp_l2d);
13291 
13292 %}
13293 
13294 // ============================================================================
13295 // clearing of an array
13296 
13297 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
13298 %{
13299   match(Set dummy (ClearArray cnt base));
13300   effect(USE_KILL cnt, USE_KILL base);
13301 
13302   ins_cost(4 * INSN_COST);
13303   format %{ "ClearArray $cnt, $base" %}
13304 
13305   ins_encode %{
13306     __ zero_words($base$$Register, $cnt$$Register);
13307   %}
13308 
13309   ins_pipe(pipe_class_memory);
13310 %}
13311 
13312 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
13313 %{
13314   predicate((u_int64_t)n->in(2)->get_long()
13315             < (u_int64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
13316   match(Set dummy (ClearArray cnt base));
13317   effect(USE_KILL base);
13318 
13319   ins_cost(4 * INSN_COST);
13320   format %{ "ClearArray $cnt, $base" %}
13321 
13322   ins_encode %{
13323     __ zero_words($base$$Register, (u_int64_t)$cnt$$constant);
13324   %}
13325 
13326   ins_pipe(pipe_class_memory);
13327 %}
13328 
13329 // ============================================================================
13330 // Overflow Math Instructions
13331 
13332 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13333 %{
13334   match(Set cr (OverflowAddI op1 op2));
13335 
13336   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
13337   ins_cost(INSN_COST);
13338   ins_encode %{
13339     __ cmnw($op1$$Register, $op2$$Register);
13340   %}
13341 
13342   ins_pipe(icmp_reg_reg);
13343 %}
13344 
13345 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
13346 %{
13347   match(Set cr (OverflowAddI op1 op2));
13348 
13349   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
13350   ins_cost(INSN_COST);
13351   ins_encode %{
13352     __ cmnw($op1$$Register, $op2$$constant);
13353   %}
13354 
13355   ins_pipe(icmp_reg_imm);
13356 %}
13357 
13358 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13359 %{
13360   match(Set cr (OverflowAddL op1 op2));
13361 
13362   format %{ "cmn   $op1, $op2\t# overflow check long" %}
13363   ins_cost(INSN_COST);
13364   ins_encode %{
13365     __ cmn($op1$$Register, $op2$$Register);
13366   %}
13367 
13368   ins_pipe(icmp_reg_reg);
13369 %}
13370 
13371 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
13372 %{
13373   match(Set cr (OverflowAddL op1 op2));
13374 
13375   format %{ "cmn   $op1, $op2\t# overflow check long" %}
13376   ins_cost(INSN_COST);
13377   ins_encode %{
13378     __ cmn($op1$$Register, $op2$$constant);
13379   %}
13380 
13381   ins_pipe(icmp_reg_imm);
13382 %}
13383 
13384 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13385 %{
13386   match(Set cr (OverflowSubI op1 op2));
13387 
13388   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
13389   ins_cost(INSN_COST);
13390   ins_encode %{
13391     __ cmpw($op1$$Register, $op2$$Register);
13392   %}
13393 
13394   ins_pipe(icmp_reg_reg);
13395 %}
13396 
13397 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
13398 %{
13399   match(Set cr (OverflowSubI op1 op2));
13400 
13401   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
13402   ins_cost(INSN_COST);
13403   ins_encode %{
13404     __ cmpw($op1$$Register, $op2$$constant);
13405   %}
13406 
13407   ins_pipe(icmp_reg_imm);
13408 %}
13409 
13410 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13411 %{
13412   match(Set cr (OverflowSubL op1 op2));
13413 
13414   format %{ "cmp   $op1, $op2\t# overflow check long" %}
13415   ins_cost(INSN_COST);
13416   ins_encode %{
13417     __ cmp($op1$$Register, $op2$$Register);
13418   %}
13419 
13420   ins_pipe(icmp_reg_reg);
13421 %}
13422 
13423 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
13424 %{
13425   match(Set cr (OverflowSubL op1 op2));
13426 
13427   format %{ "cmp   $op1, $op2\t# overflow check long" %}
13428   ins_cost(INSN_COST);
13429   ins_encode %{
13430     __ subs(zr, $op1$$Register, $op2$$constant);
13431   %}
13432 
13433   ins_pipe(icmp_reg_imm);
13434 %}
13435 
13436 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
13437 %{
13438   match(Set cr (OverflowSubI zero op1));
13439 
13440   format %{ "cmpw  zr, $op1\t# overflow check int" %}
13441   ins_cost(INSN_COST);
13442   ins_encode %{
13443     __ cmpw(zr, $op1$$Register);
13444   %}
13445 
13446   ins_pipe(icmp_reg_imm);
13447 %}
13448 
13449 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
13450 %{
13451   match(Set cr (OverflowSubL zero op1));
13452 
13453   format %{ "cmp   zr, $op1\t# overflow check long" %}
13454   ins_cost(INSN_COST);
13455   ins_encode %{
13456     __ cmp(zr, $op1$$Register);
13457   %}
13458 
13459   ins_pipe(icmp_reg_imm);
13460 %}
13461 
13462 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
13463 %{
13464   match(Set cr (OverflowMulI op1 op2));
13465 
13466   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
13467             "cmp   rscratch1, rscratch1, sxtw\n\t"
13468             "movw  rscratch1, #0x80000000\n\t"
13469             "cselw rscratch1, rscratch1, zr, NE\n\t"
13470             "cmpw  rscratch1, #1" %}
13471   ins_cost(5 * INSN_COST);
13472   ins_encode %{
13473     __ smull(rscratch1, $op1$$Register, $op2$$Register);
13474     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
13475     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
13476     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
13477     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
13478   %}
13479 
13480   ins_pipe(pipe_slow);
13481 %}
13482 
13483 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
13484 %{
13485   match(If cmp (OverflowMulI op1 op2));
13486   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
13487             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
13488   effect(USE labl, KILL cr);
13489 
13490   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
13491             "cmp   rscratch1, rscratch1, sxtw\n\t"
13492             "b$cmp   $labl" %}
13493   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
13494   ins_encode %{
13495     Label* L = $labl$$label;
13496     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13497     __ smull(rscratch1, $op1$$Register, $op2$$Register);
13498     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
13499     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
13500   %}
13501 
13502   ins_pipe(pipe_serial);
13503 %}
13504 
13505 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13506 %{
13507   match(Set cr (OverflowMulL op1 op2));
13508 
13509   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
13510             "smulh rscratch2, $op1, $op2\n\t"
13511             "cmp   rscratch2, rscratch1, ASR #63\n\t"
13512             "movw  rscratch1, #0x80000000\n\t"
13513             "cselw rscratch1, rscratch1, zr, NE\n\t"
13514             "cmpw  rscratch1, #1" %}
13515   ins_cost(6 * INSN_COST);
13516   ins_encode %{
13517     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
13518     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
13519     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
13520     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
13521     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
13522     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
13523   %}
13524 
13525   ins_pipe(pipe_slow);
13526 %}
13527 
13528 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
13529 %{
13530   match(If cmp (OverflowMulL op1 op2));
13531   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
13532             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
13533   effect(USE labl, KILL cr);
13534 
13535   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
13536             "smulh rscratch2, $op1, $op2\n\t"
13537             "cmp   rscratch2, rscratch1, ASR #63\n\t"
13538             "b$cmp $labl" %}
13539   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
13540   ins_encode %{
13541     Label* L = $labl$$label;
13542     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13543     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
13544     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
13545     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
13546     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
13547   %}
13548 
13549   ins_pipe(pipe_serial);
13550 %}
13551 
13552 // ============================================================================
13553 // Compare Instructions
13554 
13555 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
13556 %{
13557   match(Set cr (CmpI op1 op2));
13558 
13559   effect(DEF cr, USE op1, USE op2);
13560 
13561   ins_cost(INSN_COST);
13562   format %{ "cmpw  $op1, $op2" %}
13563 
13564   ins_encode(aarch64_enc_cmpw(op1, op2));
13565 
13566   ins_pipe(icmp_reg_reg);
13567 %}
13568 
13569 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
13570 %{
13571   match(Set cr (CmpI op1 zero));
13572 
13573   effect(DEF cr, USE op1);
13574 
13575   ins_cost(INSN_COST);
13576   format %{ "cmpw $op1, 0" %}
13577 
13578   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
13579 
13580   ins_pipe(icmp_reg_imm);
13581 %}
13582 
13583 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
13584 %{
13585   match(Set cr (CmpI op1 op2));
13586 
13587   effect(DEF cr, USE op1);
13588 
13589   ins_cost(INSN_COST);
13590   format %{ "cmpw  $op1, $op2" %}
13591 
13592   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
13593 
13594   ins_pipe(icmp_reg_imm);
13595 %}
13596 
13597 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
13598 %{
13599   match(Set cr (CmpI op1 op2));
13600 
13601   effect(DEF cr, USE op1);
13602 
13603   ins_cost(INSN_COST * 2);
13604   format %{ "cmpw  $op1, $op2" %}
13605 
13606   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
13607 
13608   ins_pipe(icmp_reg_imm);
13609 %}
13610 
13611 // Unsigned compare Instructions; really, same as signed compare
13612 // except it should only be used to feed an If or a CMovI which takes a
13613 // cmpOpU.
13614 
13615 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
13616 %{
13617   match(Set cr (CmpU op1 op2));
13618 
13619   effect(DEF cr, USE op1, USE op2);
13620 
13621   ins_cost(INSN_COST);
13622   format %{ "cmpw  $op1, $op2\t# unsigned" %}
13623 
13624   ins_encode(aarch64_enc_cmpw(op1, op2));
13625 
13626   ins_pipe(icmp_reg_reg);
13627 %}
13628 
13629 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
13630 %{
13631   match(Set cr (CmpU op1 zero));
13632 
13633   effect(DEF cr, USE op1);
13634 
13635   ins_cost(INSN_COST);
13636   format %{ "cmpw $op1, #0\t# unsigned" %}
13637 
13638   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
13639 
13640   ins_pipe(icmp_reg_imm);
13641 %}
13642 
13643 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
13644 %{
13645   match(Set cr (CmpU op1 op2));
13646 
13647   effect(DEF cr, USE op1);
13648 
13649   ins_cost(INSN_COST);
13650   format %{ "cmpw  $op1, $op2\t# unsigned" %}
13651 
13652   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
13653 
13654   ins_pipe(icmp_reg_imm);
13655 %}
13656 
13657 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
13658 %{
13659   match(Set cr (CmpU op1 op2));
13660 
13661   effect(DEF cr, USE op1);
13662 
13663   ins_cost(INSN_COST * 2);
13664   format %{ "cmpw  $op1, $op2\t# unsigned" %}
13665 
13666   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
13667 
13668   ins_pipe(icmp_reg_imm);
13669 %}
13670 
13671 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
13672 %{
13673   match(Set cr (CmpL op1 op2));
13674 
13675   effect(DEF cr, USE op1, USE op2);
13676 
13677   ins_cost(INSN_COST);
13678   format %{ "cmp  $op1, $op2" %}
13679 
13680   ins_encode(aarch64_enc_cmp(op1, op2));
13681 
13682   ins_pipe(icmp_reg_reg);
13683 %}
13684 
13685 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
13686 %{
13687   match(Set cr (CmpL op1 zero));
13688 
13689   effect(DEF cr, USE op1);
13690 
13691   ins_cost(INSN_COST);
13692   format %{ "tst  $op1" %}
13693 
13694   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
13695 
13696   ins_pipe(icmp_reg_imm);
13697 %}
13698 
13699 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
13700 %{
13701   match(Set cr (CmpL op1 op2));
13702 
13703   effect(DEF cr, USE op1);
13704 
13705   ins_cost(INSN_COST);
13706   format %{ "cmp  $op1, $op2" %}
13707 
13708   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
13709 
13710   ins_pipe(icmp_reg_imm);
13711 %}
13712 
13713 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
13714 %{
13715   match(Set cr (CmpL op1 op2));
13716 
13717   effect(DEF cr, USE op1);
13718 
13719   ins_cost(INSN_COST * 2);
13720   format %{ "cmp  $op1, $op2" %}
13721 
13722   ins_encode(aarch64_enc_cmp_imm(op1, op2));
13723 
13724   ins_pipe(icmp_reg_imm);
13725 %}
13726 
13727 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
13728 %{
13729   match(Set cr (CmpUL op1 op2));
13730 
13731   effect(DEF cr, USE op1, USE op2);
13732 
13733   ins_cost(INSN_COST);
13734   format %{ "cmp  $op1, $op2" %}
13735 
13736   ins_encode(aarch64_enc_cmp(op1, op2));
13737 
13738   ins_pipe(icmp_reg_reg);
13739 %}
13740 
13741 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
13742 %{
13743   match(Set cr (CmpUL op1 zero));
13744 
13745   effect(DEF cr, USE op1);
13746 
13747   ins_cost(INSN_COST);
13748   format %{ "tst  $op1" %}
13749 
13750   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
13751 
13752   ins_pipe(icmp_reg_imm);
13753 %}
13754 
13755 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
13756 %{
13757   match(Set cr (CmpUL op1 op2));
13758 
13759   effect(DEF cr, USE op1);
13760 
13761   ins_cost(INSN_COST);
13762   format %{ "cmp  $op1, $op2" %}
13763 
13764   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
13765 
13766   ins_pipe(icmp_reg_imm);
13767 %}
13768 
13769 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
13770 %{
13771   match(Set cr (CmpUL op1 op2));
13772 
13773   effect(DEF cr, USE op1);
13774 
13775   ins_cost(INSN_COST * 2);
13776   format %{ "cmp  $op1, $op2" %}
13777 
13778   ins_encode(aarch64_enc_cmp_imm(op1, op2));
13779 
13780   ins_pipe(icmp_reg_imm);
13781 %}
13782 
13783 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
13784 %{
13785   match(Set cr (CmpP op1 op2));
13786 
13787   effect(DEF cr, USE op1, USE op2);
13788 
13789   ins_cost(INSN_COST);
13790   format %{ "cmp  $op1, $op2\t // ptr" %}
13791 
13792   ins_encode(aarch64_enc_cmpp(op1, op2));
13793 
13794   ins_pipe(icmp_reg_reg);
13795 %}
13796 
13797 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
13798 %{
13799   match(Set cr (CmpN op1 op2));
13800 
13801   effect(DEF cr, USE op1, USE op2);
13802 
13803   ins_cost(INSN_COST);
13804   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
13805 
13806   ins_encode(aarch64_enc_cmpn(op1, op2));
13807 
13808   ins_pipe(icmp_reg_reg);
13809 %}
13810 
13811 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
13812 %{
13813   match(Set cr (CmpP op1 zero));
13814 
13815   effect(DEF cr, USE op1, USE zero);
13816 
13817   ins_cost(INSN_COST);
13818   format %{ "cmp  $op1, 0\t // ptr" %}
13819 
13820   ins_encode(aarch64_enc_testp(op1));
13821 
13822   ins_pipe(icmp_reg_imm);
13823 %}
13824 
13825 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
13826 %{
13827   match(Set cr (CmpN op1 zero));
13828 
13829   effect(DEF cr, USE op1, USE zero);
13830 
13831   ins_cost(INSN_COST);
13832   format %{ "cmp  $op1, 0\t // compressed ptr" %}
13833 
13834   ins_encode(aarch64_enc_testn(op1));
13835 
13836   ins_pipe(icmp_reg_imm);
13837 %}
13838 
13839 // FP comparisons
13840 //
13841 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
13842 // using normal cmpOp. See declaration of rFlagsReg for details.
13843 
13844 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
13845 %{
13846   match(Set cr (CmpF src1 src2));
13847 
13848   ins_cost(3 * INSN_COST);
13849   format %{ "fcmps $src1, $src2" %}
13850 
13851   ins_encode %{
13852     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
13853   %}
13854 
13855   ins_pipe(pipe_class_compare);
13856 %}
13857 
13858 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
13859 %{
13860   match(Set cr (CmpF src1 src2));
13861 
13862   ins_cost(3 * INSN_COST);
13863   format %{ "fcmps $src1, 0.0" %}
13864 
13865   ins_encode %{
13866     __ fcmps(as_FloatRegister($src1$$reg), 0.0D);
13867   %}
13868 
13869   ins_pipe(pipe_class_compare);
13870 %}
13871 // FROM HERE
13872 
13873 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
13874 %{
13875   match(Set cr (CmpD src1 src2));
13876 
13877   ins_cost(3 * INSN_COST);
13878   format %{ "fcmpd $src1, $src2" %}
13879 
13880   ins_encode %{
13881     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
13882   %}
13883 
13884   ins_pipe(pipe_class_compare);
13885 %}
13886 
13887 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
13888 %{
13889   match(Set cr (CmpD src1 src2));
13890 
13891   ins_cost(3 * INSN_COST);
13892   format %{ "fcmpd $src1, 0.0" %}
13893 
13894   ins_encode %{
13895     __ fcmpd(as_FloatRegister($src1$$reg), 0.0D);
13896   %}
13897 
13898   ins_pipe(pipe_class_compare);
13899 %}
13900 
13901 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
13902 %{
13903   match(Set dst (CmpF3 src1 src2));
13904   effect(KILL cr);
13905 
13906   ins_cost(5 * INSN_COST);
13907   format %{ "fcmps $src1, $src2\n\t"
13908             "csinvw($dst, zr, zr, eq\n\t"
13909             "csnegw($dst, $dst, $dst, lt)"
13910   %}
13911 
13912   ins_encode %{
13913     Label done;
13914     FloatRegister s1 = as_FloatRegister($src1$$reg);
13915     FloatRegister s2 = as_FloatRegister($src2$$reg);
13916     Register d = as_Register($dst$$reg);
13917     __ fcmps(s1, s2);
13918     // installs 0 if EQ else -1
13919     __ csinvw(d, zr, zr, Assembler::EQ);
13920     // keeps -1 if less or unordered else installs 1
13921     __ csnegw(d, d, d, Assembler::LT);
13922     __ bind(done);
13923   %}
13924 
13925   ins_pipe(pipe_class_default);
13926 
13927 %}
13928 
13929 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
13930 %{
13931   match(Set dst (CmpD3 src1 src2));
13932   effect(KILL cr);
13933 
13934   ins_cost(5 * INSN_COST);
13935   format %{ "fcmpd $src1, $src2\n\t"
13936             "csinvw($dst, zr, zr, eq\n\t"
13937             "csnegw($dst, $dst, $dst, lt)"
13938   %}
13939 
13940   ins_encode %{
13941     Label done;
13942     FloatRegister s1 = as_FloatRegister($src1$$reg);
13943     FloatRegister s2 = as_FloatRegister($src2$$reg);
13944     Register d = as_Register($dst$$reg);
13945     __ fcmpd(s1, s2);
13946     // installs 0 if EQ else -1
13947     __ csinvw(d, zr, zr, Assembler::EQ);
13948     // keeps -1 if less or unordered else installs 1
13949     __ csnegw(d, d, d, Assembler::LT);
13950     __ bind(done);
13951   %}
13952   ins_pipe(pipe_class_default);
13953 
13954 %}
13955 
13956 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
13957 %{
13958   match(Set dst (CmpF3 src1 zero));
13959   effect(KILL cr);
13960 
13961   ins_cost(5 * INSN_COST);
13962   format %{ "fcmps $src1, 0.0\n\t"
13963             "csinvw($dst, zr, zr, eq\n\t"
13964             "csnegw($dst, $dst, $dst, lt)"
13965   %}
13966 
13967   ins_encode %{
13968     Label done;
13969     FloatRegister s1 = as_FloatRegister($src1$$reg);
13970     Register d = as_Register($dst$$reg);
13971     __ fcmps(s1, 0.0D);
13972     // installs 0 if EQ else -1
13973     __ csinvw(d, zr, zr, Assembler::EQ);
13974     // keeps -1 if less or unordered else installs 1
13975     __ csnegw(d, d, d, Assembler::LT);
13976     __ bind(done);
13977   %}
13978 
13979   ins_pipe(pipe_class_default);
13980 
13981 %}
13982 
13983 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
13984 %{
13985   match(Set dst (CmpD3 src1 zero));
13986   effect(KILL cr);
13987 
13988   ins_cost(5 * INSN_COST);
13989   format %{ "fcmpd $src1, 0.0\n\t"
13990             "csinvw($dst, zr, zr, eq\n\t"
13991             "csnegw($dst, $dst, $dst, lt)"
13992   %}
13993 
13994   ins_encode %{
13995     Label done;
13996     FloatRegister s1 = as_FloatRegister($src1$$reg);
13997     Register d = as_Register($dst$$reg);
13998     __ fcmpd(s1, 0.0D);
13999     // installs 0 if EQ else -1
14000     __ csinvw(d, zr, zr, Assembler::EQ);
14001     // keeps -1 if less or unordered else installs 1
14002     __ csnegw(d, d, d, Assembler::LT);
14003     __ bind(done);
14004   %}
14005   ins_pipe(pipe_class_default);
14006 
14007 %}
14008 
14009 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14010 %{
14011   match(Set dst (CmpLTMask p q));
14012   effect(KILL cr);
14013 
14014   ins_cost(3 * INSN_COST);
14015 
14016   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14017             "csetw $dst, lt\n\t"
14018             "subw $dst, zr, $dst"
14019   %}
14020 
14021   ins_encode %{
14022     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14023     __ csetw(as_Register($dst$$reg), Assembler::LT);
14024     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14025   %}
14026 
14027   ins_pipe(ialu_reg_reg);
14028 %}
14029 
14030 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14031 %{
14032   match(Set dst (CmpLTMask src zero));
14033   effect(KILL cr);
14034 
14035   ins_cost(INSN_COST);
14036 
14037   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14038 
14039   ins_encode %{
14040     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14041   %}
14042 
14043   ins_pipe(ialu_reg_shift);
14044 %}
14045 
14046 // ============================================================================
14047 // Max and Min
14048 
14049 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14050 %{
14051   effect( DEF dst, USE src1, USE src2, USE cr );
14052 
14053   ins_cost(INSN_COST * 2);
14054   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
14055 
14056   ins_encode %{
14057     __ cselw(as_Register($dst$$reg),
14058              as_Register($src1$$reg),
14059              as_Register($src2$$reg),
14060              Assembler::LT);
14061   %}
14062 
14063   ins_pipe(icond_reg_reg);
14064 %}
14065 
14066 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
14067 %{
14068   match(Set dst (MinI src1 src2));
14069   ins_cost(INSN_COST * 3);
14070 
14071   expand %{
14072     rFlagsReg cr;
14073     compI_reg_reg(cr, src1, src2);
14074     cmovI_reg_reg_lt(dst, src1, src2, cr);
14075   %}
14076 
14077 %}
14078 // FROM HERE
14079 
14080 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14081 %{
14082   effect( DEF dst, USE src1, USE src2, USE cr );
14083 
14084   ins_cost(INSN_COST * 2);
14085   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
14086 
14087   ins_encode %{
14088     __ cselw(as_Register($dst$$reg),
14089              as_Register($src1$$reg),
14090              as_Register($src2$$reg),
14091              Assembler::GT);
14092   %}
14093 
14094   ins_pipe(icond_reg_reg);
14095 %}
14096 
14097 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
14098 %{
14099   match(Set dst (MaxI src1 src2));
14100   ins_cost(INSN_COST * 3);
14101   expand %{
14102     rFlagsReg cr;
14103     compI_reg_reg(cr, src1, src2);
14104     cmovI_reg_reg_gt(dst, src1, src2, cr);
14105   %}
14106 %}
14107 
14108 // ============================================================================
14109 // Branch Instructions
14110 
14111 // Direct Branch.
14112 instruct branch(label lbl)
14113 %{
14114   match(Goto);
14115 
14116   effect(USE lbl);
14117 
14118   ins_cost(BRANCH_COST);
14119   format %{ "b  $lbl" %}
14120 
14121   ins_encode(aarch64_enc_b(lbl));
14122 
14123   ins_pipe(pipe_branch);
14124 %}
14125 
14126 // Conditional Near Branch
14127 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14128 %{
14129   // Same match rule as `branchConFar'.
14130   match(If cmp cr);
14131 
14132   effect(USE lbl);
14133 
14134   ins_cost(BRANCH_COST);
14135   // If set to 1 this indicates that the current instruction is a
14136   // short variant of a long branch. This avoids using this
14137   // instruction in first-pass matching. It will then only be used in
14138   // the `Shorten_branches' pass.
14139   // ins_short_branch(1);
14140   format %{ "b$cmp  $lbl" %}
14141 
14142   ins_encode(aarch64_enc_br_con(cmp, lbl));
14143 
14144   ins_pipe(pipe_branch_cond);
14145 %}
14146 
14147 // Conditional Near Branch Unsigned
14148 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14149 %{
14150   // Same match rule as `branchConFar'.
14151   match(If cmp cr);
14152 
14153   effect(USE lbl);
14154 
14155   ins_cost(BRANCH_COST);
14156   // If set to 1 this indicates that the current instruction is a
14157   // short variant of a long branch. This avoids using this
14158   // instruction in first-pass matching. It will then only be used in
14159   // the `Shorten_branches' pass.
14160   // ins_short_branch(1);
14161   format %{ "b$cmp  $lbl\t# unsigned" %}
14162 
14163   ins_encode(aarch64_enc_br_conU(cmp, lbl));
14164 
14165   ins_pipe(pipe_branch_cond);
14166 %}
14167 
14168 // Make use of CBZ and CBNZ.  These instructions, as well as being
14169 // shorter than (cmp; branch), have the additional benefit of not
14170 // killing the flags.
14171 
14172 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14173   match(If cmp (CmpI op1 op2));
14174   effect(USE labl);
14175 
14176   ins_cost(BRANCH_COST);
14177   format %{ "cbw$cmp   $op1, $labl" %}
14178   ins_encode %{
14179     Label* L = $labl$$label;
14180     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14181     if (cond == Assembler::EQ)
14182       __ cbzw($op1$$Register, *L);
14183     else
14184       __ cbnzw($op1$$Register, *L);
14185   %}
14186   ins_pipe(pipe_cmp_branch);
14187 %}
14188 
14189 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14190   match(If cmp (CmpL op1 op2));
14191   effect(USE labl);
14192 
14193   ins_cost(BRANCH_COST);
14194   format %{ "cb$cmp   $op1, $labl" %}
14195   ins_encode %{
14196     Label* L = $labl$$label;
14197     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14198     if (cond == Assembler::EQ)
14199       __ cbz($op1$$Register, *L);
14200     else
14201       __ cbnz($op1$$Register, *L);
14202   %}
14203   ins_pipe(pipe_cmp_branch);
14204 %}
14205 
14206 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14207   match(If cmp (CmpP op1 op2));
14208   effect(USE labl);
14209 
14210   ins_cost(BRANCH_COST);
14211   format %{ "cb$cmp   $op1, $labl" %}
14212   ins_encode %{
14213     Label* L = $labl$$label;
14214     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14215     if (cond == Assembler::EQ)
14216       __ cbz($op1$$Register, *L);
14217     else
14218       __ cbnz($op1$$Register, *L);
14219   %}
14220   ins_pipe(pipe_cmp_branch);
14221 %}
14222 
14223 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14224   match(If cmp (CmpN op1 op2));
14225   effect(USE labl);
14226 
14227   ins_cost(BRANCH_COST);
14228   format %{ "cbw$cmp   $op1, $labl" %}
14229   ins_encode %{
14230     Label* L = $labl$$label;
14231     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14232     if (cond == Assembler::EQ)
14233       __ cbzw($op1$$Register, *L);
14234     else
14235       __ cbnzw($op1$$Register, *L);
14236   %}
14237   ins_pipe(pipe_cmp_branch);
14238 %}
14239 
14240 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
14241   match(If cmp (CmpP (DecodeN oop) zero));
14242   effect(USE labl);
14243 
14244   ins_cost(BRANCH_COST);
14245   format %{ "cb$cmp   $oop, $labl" %}
14246   ins_encode %{
14247     Label* L = $labl$$label;
14248     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14249     if (cond == Assembler::EQ)
14250       __ cbzw($oop$$Register, *L);
14251     else
14252       __ cbnzw($oop$$Register, *L);
14253   %}
14254   ins_pipe(pipe_cmp_branch);
14255 %}
14256 
14257 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
14258   match(If cmp (CmpU op1 op2));
14259   effect(USE labl);
14260 
14261   ins_cost(BRANCH_COST);
14262   format %{ "cbw$cmp   $op1, $labl" %}
14263   ins_encode %{
14264     Label* L = $labl$$label;
14265     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14266     if (cond == Assembler::EQ || cond == Assembler::LS)
14267       __ cbzw($op1$$Register, *L);
14268     else
14269       __ cbnzw($op1$$Register, *L);
14270   %}
14271   ins_pipe(pipe_cmp_branch);
14272 %}
14273 
14274 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
14275   match(If cmp (CmpUL op1 op2));
14276   effect(USE labl);
14277 
14278   ins_cost(BRANCH_COST);
14279   format %{ "cb$cmp   $op1, $labl" %}
14280   ins_encode %{
14281     Label* L = $labl$$label;
14282     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14283     if (cond == Assembler::EQ || cond == Assembler::LS)
14284       __ cbz($op1$$Register, *L);
14285     else
14286       __ cbnz($op1$$Register, *L);
14287   %}
14288   ins_pipe(pipe_cmp_branch);
14289 %}
14290 
14291 // Test bit and Branch
14292 
14293 // Patterns for short (< 32KiB) variants
14294 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
14295   match(If cmp (CmpL op1 op2));
14296   effect(USE labl);
14297 
14298   ins_cost(BRANCH_COST);
14299   format %{ "cb$cmp   $op1, $labl # long" %}
14300   ins_encode %{
14301     Label* L = $labl$$label;
14302     Assembler::Condition cond =
14303       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14304     __ tbr(cond, $op1$$Register, 63, *L);
14305   %}
14306   ins_pipe(pipe_cmp_branch);
14307   ins_short_branch(1);
14308 %}
14309 
14310 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
14311   match(If cmp (CmpI op1 op2));
14312   effect(USE labl);
14313 
14314   ins_cost(BRANCH_COST);
14315   format %{ "cb$cmp   $op1, $labl # int" %}
14316   ins_encode %{
14317     Label* L = $labl$$label;
14318     Assembler::Condition cond =
14319       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14320     __ tbr(cond, $op1$$Register, 31, *L);
14321   %}
14322   ins_pipe(pipe_cmp_branch);
14323   ins_short_branch(1);
14324 %}
14325 
14326 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
14327   match(If cmp (CmpL (AndL op1 op2) op3));
14328   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
14329   effect(USE labl);
14330 
14331   ins_cost(BRANCH_COST);
14332   format %{ "tb$cmp   $op1, $op2, $labl" %}
14333   ins_encode %{
14334     Label* L = $labl$$label;
14335     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14336     int bit = exact_log2($op2$$constant);
14337     __ tbr(cond, $op1$$Register, bit, *L);
14338   %}
14339   ins_pipe(pipe_cmp_branch);
14340   ins_short_branch(1);
14341 %}
14342 
14343 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
14344   match(If cmp (CmpI (AndI op1 op2) op3));
14345   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
14346   effect(USE labl);
14347 
14348   ins_cost(BRANCH_COST);
14349   format %{ "tb$cmp   $op1, $op2, $labl" %}
14350   ins_encode %{
14351     Label* L = $labl$$label;
14352     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14353     int bit = exact_log2($op2$$constant);
14354     __ tbr(cond, $op1$$Register, bit, *L);
14355   %}
14356   ins_pipe(pipe_cmp_branch);
14357   ins_short_branch(1);
14358 %}
14359 
14360 // And far variants
14361 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
14362   match(If cmp (CmpL op1 op2));
14363   effect(USE labl);
14364 
14365   ins_cost(BRANCH_COST);
14366   format %{ "cb$cmp   $op1, $labl # long" %}
14367   ins_encode %{
14368     Label* L = $labl$$label;
14369     Assembler::Condition cond =
14370       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14371     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
14372   %}
14373   ins_pipe(pipe_cmp_branch);
14374 %}
14375 
14376 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
14377   match(If cmp (CmpI op1 op2));
14378   effect(USE labl);
14379 
14380   ins_cost(BRANCH_COST);
14381   format %{ "cb$cmp   $op1, $labl # int" %}
14382   ins_encode %{
14383     Label* L = $labl$$label;
14384     Assembler::Condition cond =
14385       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
14386     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
14387   %}
14388   ins_pipe(pipe_cmp_branch);
14389 %}
14390 
14391 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
14392   match(If cmp (CmpL (AndL op1 op2) op3));
14393   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
14394   effect(USE labl);
14395 
14396   ins_cost(BRANCH_COST);
14397   format %{ "tb$cmp   $op1, $op2, $labl" %}
14398   ins_encode %{
14399     Label* L = $labl$$label;
14400     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14401     int bit = exact_log2($op2$$constant);
14402     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
14403   %}
14404   ins_pipe(pipe_cmp_branch);
14405 %}
14406 
14407 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
14408   match(If cmp (CmpI (AndI op1 op2) op3));
14409   predicate(is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
14410   effect(USE labl);
14411 
14412   ins_cost(BRANCH_COST);
14413   format %{ "tb$cmp   $op1, $op2, $labl" %}
14414   ins_encode %{
14415     Label* L = $labl$$label;
14416     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14417     int bit = exact_log2($op2$$constant);
14418     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
14419   %}
14420   ins_pipe(pipe_cmp_branch);
14421 %}
14422 
14423 // Test bits
14424 
14425 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
14426   match(Set cr (CmpL (AndL op1 op2) op3));
14427   predicate(Assembler::operand_valid_for_logical_immediate
14428             (/*is_32*/false, n->in(1)->in(2)->get_long()));
14429 
14430   ins_cost(INSN_COST);
14431   format %{ "tst $op1, $op2 # long" %}
14432   ins_encode %{
14433     __ tst($op1$$Register, $op2$$constant);
14434   %}
14435   ins_pipe(ialu_reg_reg);
14436 %}
14437 
14438 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
14439   match(Set cr (CmpI (AndI op1 op2) op3));
14440   predicate(Assembler::operand_valid_for_logical_immediate
14441             (/*is_32*/true, n->in(1)->in(2)->get_int()));
14442 
14443   ins_cost(INSN_COST);
14444   format %{ "tst $op1, $op2 # int" %}
14445   ins_encode %{
14446     __ tstw($op1$$Register, $op2$$constant);
14447   %}
14448   ins_pipe(ialu_reg_reg);
14449 %}
14450 
14451 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
14452   match(Set cr (CmpL (AndL op1 op2) op3));
14453 
14454   ins_cost(INSN_COST);
14455   format %{ "tst $op1, $op2 # long" %}
14456   ins_encode %{
14457     __ tst($op1$$Register, $op2$$Register);
14458   %}
14459   ins_pipe(ialu_reg_reg);
14460 %}
14461 
14462 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
14463   match(Set cr (CmpI (AndI op1 op2) op3));
14464 
14465   ins_cost(INSN_COST);
14466   format %{ "tstw $op1, $op2 # int" %}
14467   ins_encode %{
14468     __ tstw($op1$$Register, $op2$$Register);
14469   %}
14470   ins_pipe(ialu_reg_reg);
14471 %}
14472 
14473 
14474 // Conditional Far Branch
14475 // Conditional Far Branch Unsigned
14476 // TODO: fixme
14477 
14478 // counted loop end branch near
14479 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
14480 %{
14481   match(CountedLoopEnd cmp cr);
14482 
14483   effect(USE lbl);
14484 
14485   ins_cost(BRANCH_COST);
14486   // short variant.
14487   // ins_short_branch(1);
14488   format %{ "b$cmp $lbl \t// counted loop end" %}
14489 
14490   ins_encode(aarch64_enc_br_con(cmp, lbl));
14491 
14492   ins_pipe(pipe_branch);
14493 %}
14494 
14495 // counted loop end branch near Unsigned
14496 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14497 %{
14498   match(CountedLoopEnd cmp cr);
14499 
14500   effect(USE lbl);
14501 
14502   ins_cost(BRANCH_COST);
14503   // short variant.
14504   // ins_short_branch(1);
14505   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
14506 
14507   ins_encode(aarch64_enc_br_conU(cmp, lbl));
14508 
14509   ins_pipe(pipe_branch);
14510 %}
14511 
14512 // counted loop end branch far
14513 // counted loop end branch far unsigned
14514 // TODO: fixme
14515 
14516 // ============================================================================
14517 // inlined locking and unlocking
14518 
14519 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
14520 %{
14521   match(Set cr (FastLock object box));
14522   effect(TEMP tmp, TEMP tmp2);
14523 
14524   // TODO
14525   // identify correct cost
14526   ins_cost(5 * INSN_COST);
14527   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
14528 
14529   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
14530 
14531   ins_pipe(pipe_serial);
14532 %}
14533 
14534 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
14535 %{
14536   match(Set cr (FastUnlock object box));
14537   effect(TEMP tmp, TEMP tmp2);
14538 
14539   ins_cost(5 * INSN_COST);
14540   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
14541 
14542   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
14543 
14544   ins_pipe(pipe_serial);
14545 %}
14546 
14547 
14548 // ============================================================================
14549 // Safepoint Instructions
14550 
14551 // TODO
14552 // provide a near and far version of this code
14553 
14554 instruct safePoint(iRegP poll)
14555 %{
14556   match(SafePoint poll);
14557 
14558   format %{
14559     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
14560   %}
14561   ins_encode %{
14562     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
14563   %}
14564   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
14565 %}
14566 
14567 
14568 // ============================================================================
14569 // Procedure Call/Return Instructions
14570 
14571 // Call Java Static Instruction
14572 
14573 instruct CallStaticJavaDirect(method meth)
14574 %{
14575   match(CallStaticJava);
14576 
14577   effect(USE meth);
14578 
14579   ins_cost(CALL_COST);
14580 
14581   format %{ "call,static $meth \t// ==> " %}
14582 
14583   ins_encode( aarch64_enc_java_static_call(meth),
14584               aarch64_enc_call_epilog );
14585 
14586   ins_pipe(pipe_class_call);
14587 %}
14588 
14589 // TO HERE
14590 
14591 // Call Java Dynamic Instruction
14592 instruct CallDynamicJavaDirect(method meth)
14593 %{
14594   match(CallDynamicJava);
14595 
14596   effect(USE meth);
14597 
14598   ins_cost(CALL_COST);
14599 
14600   format %{ "CALL,dynamic $meth \t// ==> " %}
14601 
14602   ins_encode( aarch64_enc_java_dynamic_call(meth),
14603                aarch64_enc_call_epilog );
14604 
14605   ins_pipe(pipe_class_call);
14606 %}
14607 
14608 // Call Runtime Instruction
14609 
14610 instruct CallRuntimeDirect(method meth)
14611 %{
14612   match(CallRuntime);
14613 
14614   effect(USE meth);
14615 
14616   ins_cost(CALL_COST);
14617 
14618   format %{ "CALL, runtime $meth" %}
14619 
14620   ins_encode( aarch64_enc_java_to_runtime(meth) );
14621 
14622   ins_pipe(pipe_class_call);
14623 %}
14624 
14625 // Call Runtime Instruction
14626 
14627 instruct CallLeafDirect(method meth)
14628 %{
14629   match(CallLeaf);
14630 
14631   effect(USE meth);
14632 
14633   ins_cost(CALL_COST);
14634 
14635   format %{ "CALL, runtime leaf $meth" %}
14636 
14637   ins_encode( aarch64_enc_java_to_runtime(meth) );
14638 
14639   ins_pipe(pipe_class_call);
14640 %}
14641 
14642 // Call Runtime Instruction
14643 
14644 instruct CallLeafNoFPDirect(method meth)
14645 %{
14646   match(CallLeafNoFP);
14647 
14648   effect(USE meth);
14649 
14650   ins_cost(CALL_COST);
14651 
14652   format %{ "CALL, runtime leaf nofp $meth" %}
14653 
14654   ins_encode( aarch64_enc_java_to_runtime(meth) );
14655 
14656   ins_pipe(pipe_class_call);
14657 %}
14658 
14659 // Tail Call; Jump from runtime stub to Java code.
14660 // Also known as an 'interprocedural jump'.
14661 // Target of jump will eventually return to caller.
14662 // TailJump below removes the return address.
14663 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
14664 %{
14665   match(TailCall jump_target method_oop);
14666 
14667   ins_cost(CALL_COST);
14668 
14669   format %{ "br $jump_target\t# $method_oop holds method oop" %}
14670 
14671   ins_encode(aarch64_enc_tail_call(jump_target));
14672 
14673   ins_pipe(pipe_class_call);
14674 %}
14675 
14676 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
14677 %{
14678   match(TailJump jump_target ex_oop);
14679 
14680   ins_cost(CALL_COST);
14681 
14682   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
14683 
14684   ins_encode(aarch64_enc_tail_jmp(jump_target));
14685 
14686   ins_pipe(pipe_class_call);
14687 %}
14688 
14689 // Create exception oop: created by stack-crawling runtime code.
14690 // Created exception is now available to this handler, and is setup
14691 // just prior to jumping to this handler. No code emitted.
14692 // TODO check
14693 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
14694 instruct CreateException(iRegP_R0 ex_oop)
14695 %{
14696   match(Set ex_oop (CreateEx));
14697 
14698   format %{ " -- \t// exception oop; no code emitted" %}
14699 
14700   size(0);
14701 
14702   ins_encode( /*empty*/ );
14703 
14704   ins_pipe(pipe_class_empty);
14705 %}
14706 
14707 // Rethrow exception: The exception oop will come in the first
14708 // argument position. Then JUMP (not call) to the rethrow stub code.
14709 instruct RethrowException() %{
14710   match(Rethrow);
14711   ins_cost(CALL_COST);
14712 
14713   format %{ "b rethrow_stub" %}
14714 
14715   ins_encode( aarch64_enc_rethrow() );
14716 
14717   ins_pipe(pipe_class_call);
14718 %}
14719 
14720 
14721 // Return Instruction
14722 // epilog node loads ret address into lr as part of frame pop
14723 instruct Ret()
14724 %{
14725   match(Return);
14726 
14727   format %{ "ret\t// return register" %}
14728 
14729   ins_encode( aarch64_enc_ret() );
14730 
14731   ins_pipe(pipe_branch);
14732 %}
14733 
14734 // Die now.
14735 instruct ShouldNotReachHere() %{
14736   match(Halt);
14737 
14738   ins_cost(CALL_COST);
14739   format %{ "ShouldNotReachHere" %}
14740 
14741   ins_encode %{
14742     // +1 so NativeInstruction::is_sigill_zombie_not_entrant() doesn't
14743     // return true
14744     __ dpcs1(0xdead + 1);
14745   %}
14746 
14747   ins_pipe(pipe_class_default);
14748 %}
14749 
14750 // ============================================================================
14751 // Partial Subtype Check
14752 //
14753 // superklass array for an instance of the superklass.  Set a hidden
14754 // internal cache on a hit (cache is checked with exposed code in
14755 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
14756 // encoding ALSO sets flags.
14757 
14758 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
14759 %{
14760   match(Set result (PartialSubtypeCheck sub super));
14761   effect(KILL cr, KILL temp);
14762 
14763   ins_cost(1100);  // slightly larger than the next version
14764   format %{ "partialSubtypeCheck $result, $sub, $super" %}
14765 
14766   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
14767 
14768   opcode(0x1); // Force zero of result reg on hit
14769 
14770   ins_pipe(pipe_class_memory);
14771 %}
14772 
14773 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
14774 %{
14775   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
14776   effect(KILL temp, KILL result);
14777 
14778   ins_cost(1100);  // slightly larger than the next version
14779   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
14780 
14781   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
14782 
14783   opcode(0x0); // Don't zero result reg on hit
14784 
14785   ins_pipe(pipe_class_memory);
14786 %}
14787 
14788 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14789                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
14790 %{
14791   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
14792   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14793   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14794 
14795   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
14796   ins_encode %{
14797     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
14798     __ string_compare($str1$$Register, $str2$$Register,
14799                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
14800                       $tmp1$$Register, $tmp2$$Register,
14801                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
14802   %}
14803   ins_pipe(pipe_class_memory);
14804 %}
14805 
14806 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14807                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
14808 %{
14809   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14810   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14811   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14812 
14813   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
14814   ins_encode %{
14815     __ string_compare($str1$$Register, $str2$$Register,
14816                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
14817                       $tmp1$$Register, $tmp2$$Register,
14818                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
14819   %}
14820   ins_pipe(pipe_class_memory);
14821 %}
14822 
14823 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14824                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
14825                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
14826 %{
14827   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
14828   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14829   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
14830          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14831 
14832   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
14833   ins_encode %{
14834     __ string_compare($str1$$Register, $str2$$Register,
14835                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
14836                       $tmp1$$Register, $tmp2$$Register,
14837                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
14838                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
14839   %}
14840   ins_pipe(pipe_class_memory);
14841 %}
14842 
14843 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
14844                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
14845                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
14846 %{
14847   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
14848   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14849   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
14850          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14851 
14852   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
14853   ins_encode %{
14854     __ string_compare($str1$$Register, $str2$$Register,
14855                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
14856                       $tmp1$$Register, $tmp2$$Register,
14857                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
14858                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
14859   %}
14860   ins_pipe(pipe_class_memory);
14861 %}
14862 
14863 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
14864        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
14865        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
14866 %{
14867   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
14868   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14869   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14870          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
14871   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
14872 
14873   ins_encode %{
14874     __ string_indexof($str1$$Register, $str2$$Register,
14875                       $cnt1$$Register, $cnt2$$Register,
14876                       $tmp1$$Register, $tmp2$$Register,
14877                       $tmp3$$Register, $tmp4$$Register,
14878                       $tmp5$$Register, $tmp6$$Register,
14879                       -1, $result$$Register, StrIntrinsicNode::UU);
14880   %}
14881   ins_pipe(pipe_class_memory);
14882 %}
14883 
14884 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
14885        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
14886        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
14887 %{
14888   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
14889   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14890   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14891          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
14892   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
14893 
14894   ins_encode %{
14895     __ string_indexof($str1$$Register, $str2$$Register,
14896                       $cnt1$$Register, $cnt2$$Register,
14897                       $tmp1$$Register, $tmp2$$Register,
14898                       $tmp3$$Register, $tmp4$$Register,
14899                       $tmp5$$Register, $tmp6$$Register,
14900                       -1, $result$$Register, StrIntrinsicNode::LL);
14901   %}
14902   ins_pipe(pipe_class_memory);
14903 %}
14904 
14905 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
14906        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
14907        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
14908 %{
14909   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
14910   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14911   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14912          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
14913   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
14914 
14915   ins_encode %{
14916     __ string_indexof($str1$$Register, $str2$$Register,
14917                       $cnt1$$Register, $cnt2$$Register,
14918                       $tmp1$$Register, $tmp2$$Register,
14919                       $tmp3$$Register, $tmp4$$Register,
14920                       $tmp5$$Register, $tmp6$$Register,
14921                       -1, $result$$Register, StrIntrinsicNode::UL);
14922   %}
14923   ins_pipe(pipe_class_memory);
14924 %}
14925 
14926 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14927                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14928                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14929 %{
14930   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
14931   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14932   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14933          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14934   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
14935 
14936   ins_encode %{
14937     int icnt2 = (int)$int_cnt2$$constant;
14938     __ string_indexof($str1$$Register, $str2$$Register,
14939                       $cnt1$$Register, zr,
14940                       $tmp1$$Register, $tmp2$$Register,
14941                       $tmp3$$Register, $tmp4$$Register, zr, zr,
14942                       icnt2, $result$$Register, StrIntrinsicNode::UU);
14943   %}
14944   ins_pipe(pipe_class_memory);
14945 %}
14946 
14947 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14948                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14949                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14950 %{
14951   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
14952   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14953   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14954          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14955   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
14956 
14957   ins_encode %{
14958     int icnt2 = (int)$int_cnt2$$constant;
14959     __ string_indexof($str1$$Register, $str2$$Register,
14960                       $cnt1$$Register, zr,
14961                       $tmp1$$Register, $tmp2$$Register,
14962                       $tmp3$$Register, $tmp4$$Register, zr, zr,
14963                       icnt2, $result$$Register, StrIntrinsicNode::LL);
14964   %}
14965   ins_pipe(pipe_class_memory);
14966 %}
14967 
14968 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14969                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14970                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14971 %{
14972   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
14973   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14974   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14975          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14976   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
14977 
14978   ins_encode %{
14979     int icnt2 = (int)$int_cnt2$$constant;
14980     __ string_indexof($str1$$Register, $str2$$Register,
14981                       $cnt1$$Register, zr,
14982                       $tmp1$$Register, $tmp2$$Register,
14983                       $tmp3$$Register, $tmp4$$Register, zr, zr,
14984                       icnt2, $result$$Register, StrIntrinsicNode::UL);
14985   %}
14986   ins_pipe(pipe_class_memory);
14987 %}
14988 
14989 instruct string_indexofU_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
14990                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14991                               iRegINoSp tmp3, rFlagsReg cr)
14992 %{
14993   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
14994   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
14995          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
14996 
14997   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result" %}
14998 
14999   ins_encode %{
15000     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15001                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
15002                            $tmp3$$Register);
15003   %}
15004   ins_pipe(pipe_class_memory);
15005 %}
15006 
15007 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15008                         iRegI_R0 result, rFlagsReg cr)
15009 %{
15010   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
15011   match(Set result (StrEquals (Binary str1 str2) cnt));
15012   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15013 
15014   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15015   ins_encode %{
15016     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15017     __ string_equals($str1$$Register, $str2$$Register,
15018                      $result$$Register, $cnt$$Register, 1);
15019   %}
15020   ins_pipe(pipe_class_memory);
15021 %}
15022 
15023 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
15024                         iRegI_R0 result, rFlagsReg cr)
15025 %{
15026   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
15027   match(Set result (StrEquals (Binary str1 str2) cnt));
15028   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
15029 
15030   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
15031   ins_encode %{
15032     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15033     __ string_equals($str1$$Register, $str2$$Register,
15034                      $result$$Register, $cnt$$Register, 2);
15035   %}
15036   ins_pipe(pipe_class_memory);
15037 %}
15038 
15039 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15040                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15041                        iRegP_R10 tmp, rFlagsReg cr)
15042 %{
15043   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15044   match(Set result (AryEq ary1 ary2));
15045   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15046 
15047   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
15048   ins_encode %{
15049     __ arrays_equals($ary1$$Register, $ary2$$Register,
15050                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15051                      $result$$Register, $tmp$$Register, 1);
15052     %}
15053   ins_pipe(pipe_class_memory);
15054 %}
15055 
15056 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
15057                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
15058                        iRegP_R10 tmp, rFlagsReg cr)
15059 %{
15060   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15061   match(Set result (AryEq ary1 ary2));
15062   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15063 
15064   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
15065   ins_encode %{
15066     __ arrays_equals($ary1$$Register, $ary2$$Register,
15067                      $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15068                      $result$$Register, $tmp$$Register, 2);
15069   %}
15070   ins_pipe(pipe_class_memory);
15071 %}
15072 
15073 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
15074 %{
15075   match(Set result (HasNegatives ary1 len));
15076   effect(USE_KILL ary1, USE_KILL len, KILL cr);
15077   format %{ "has negatives byte[] $ary1,$len -> $result" %}
15078   ins_encode %{
15079     __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
15080   %}
15081   ins_pipe( pipe_slow );
15082 %}
15083 
15084 // fast char[] to byte[] compression
15085 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15086                          vRegD_V0 tmp1, vRegD_V1 tmp2,
15087                          vRegD_V2 tmp3, vRegD_V3 tmp4,
15088                          iRegI_R0 result, rFlagsReg cr)
15089 %{
15090   match(Set result (StrCompressedCopy src (Binary dst len)));
15091   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15092 
15093   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
15094   ins_encode %{
15095     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15096                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
15097                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
15098                            $result$$Register);
15099   %}
15100   ins_pipe( pipe_slow );
15101 %}
15102 
15103 // fast byte[] to char[] inflation
15104 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
15105                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
15106 %{
15107   match(Set dummy (StrInflatedCopy src (Binary dst len)));
15108   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15109 
15110   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
15111   ins_encode %{
15112     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15113                           $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister, $tmp4$$Register);
15114   %}
15115   ins_pipe(pipe_class_memory);
15116 %}
15117 
15118 // encode char[] to byte[] in ISO_8859_1
15119 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
15120                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
15121                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
15122                           iRegI_R0 result, rFlagsReg cr)
15123 %{
15124   match(Set result (EncodeISOArray src (Binary dst len)));
15125   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
15126          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
15127 
15128   format %{ "Encode array $src,$dst,$len -> $result" %}
15129   ins_encode %{
15130     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15131          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
15132          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
15133   %}
15134   ins_pipe( pipe_class_memory );
15135 %}
15136 
15137 // ============================================================================
15138 // This name is KNOWN by the ADLC and cannot be changed.
15139 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
15140 // for this guy.
15141 instruct tlsLoadP(thread_RegP dst)
15142 %{
15143   match(Set dst (ThreadLocal));
15144 
15145   ins_cost(0);
15146 
15147   format %{ " -- \t// $dst=Thread::current(), empty" %}
15148 
15149   size(0);
15150 
15151   ins_encode( /*empty*/ );
15152 
15153   ins_pipe(pipe_class_empty);
15154 %}
15155 
15156 // ====================VECTOR INSTRUCTIONS=====================================
15157 
15158 // Load vector (32 bits)
15159 instruct loadV4(vecD dst, vmem4 mem)
15160 %{
15161   predicate(n->as_LoadVector()->memory_size() == 4);
15162   match(Set dst (LoadVector mem));
15163   ins_cost(4 * INSN_COST);
15164   format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
15165   ins_encode( aarch64_enc_ldrvS(dst, mem) );
15166   ins_pipe(vload_reg_mem64);
15167 %}
15168 
15169 // Load vector (64 bits)
15170 instruct loadV8(vecD dst, vmem8 mem)
15171 %{
15172   predicate(n->as_LoadVector()->memory_size() == 8);
15173   match(Set dst (LoadVector mem));
15174   ins_cost(4 * INSN_COST);
15175   format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}
15176   ins_encode( aarch64_enc_ldrvD(dst, mem) );
15177   ins_pipe(vload_reg_mem64);
15178 %}
15179 
15180 // Load Vector (128 bits)
15181 instruct loadV16(vecX dst, vmem16 mem)
15182 %{
15183   predicate(n->as_LoadVector()->memory_size() == 16);
15184   match(Set dst (LoadVector mem));
15185   ins_cost(4 * INSN_COST);
15186   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
15187   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
15188   ins_pipe(vload_reg_mem128);
15189 %}
15190 
15191 // Store Vector (32 bits)
15192 instruct storeV4(vecD src, vmem4 mem)
15193 %{
15194   predicate(n->as_StoreVector()->memory_size() == 4);
15195   match(Set mem (StoreVector mem src));
15196   ins_cost(4 * INSN_COST);
15197   format %{ "strs   $mem,$src\t# vector (32 bits)" %}
15198   ins_encode( aarch64_enc_strvS(src, mem) );
15199   ins_pipe(vstore_reg_mem64);
15200 %}
15201 
15202 // Store Vector (64 bits)
15203 instruct storeV8(vecD src, vmem8 mem)
15204 %{
15205   predicate(n->as_StoreVector()->memory_size() == 8);
15206   match(Set mem (StoreVector mem src));
15207   ins_cost(4 * INSN_COST);
15208   format %{ "strd   $mem,$src\t# vector (64 bits)" %}
15209   ins_encode( aarch64_enc_strvD(src, mem) );
15210   ins_pipe(vstore_reg_mem64);
15211 %}
15212 
15213 // Store Vector (128 bits)
15214 instruct storeV16(vecX src, vmem16 mem)
15215 %{
15216   predicate(n->as_StoreVector()->memory_size() == 16);
15217   match(Set mem (StoreVector mem src));
15218   ins_cost(4 * INSN_COST);
15219   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
15220   ins_encode( aarch64_enc_strvQ(src, mem) );
15221   ins_pipe(vstore_reg_mem128);
15222 %}
15223 
15224 instruct replicate8B(vecD dst, iRegIorL2I src)
15225 %{
15226   predicate(n->as_Vector()->length() == 4 ||
15227             n->as_Vector()->length() == 8);
15228   match(Set dst (ReplicateB src));
15229   ins_cost(INSN_COST);
15230   format %{ "dup  $dst, $src\t# vector (8B)" %}
15231   ins_encode %{
15232     __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
15233   %}
15234   ins_pipe(vdup_reg_reg64);
15235 %}
15236 
15237 instruct replicate16B(vecX dst, iRegIorL2I src)
15238 %{
15239   predicate(n->as_Vector()->length() == 16);
15240   match(Set dst (ReplicateB src));
15241   ins_cost(INSN_COST);
15242   format %{ "dup  $dst, $src\t# vector (16B)" %}
15243   ins_encode %{
15244     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
15245   %}
15246   ins_pipe(vdup_reg_reg128);
15247 %}
15248 
15249 instruct replicate8B_imm(vecD dst, immI con)
15250 %{
15251   predicate(n->as_Vector()->length() == 4 ||
15252             n->as_Vector()->length() == 8);
15253   match(Set dst (ReplicateB con));
15254   ins_cost(INSN_COST);
15255   format %{ "movi  $dst, $con\t# vector(8B)" %}
15256   ins_encode %{
15257     __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant & 0xff);
15258   %}
15259   ins_pipe(vmovi_reg_imm64);
15260 %}
15261 
15262 instruct replicate16B_imm(vecX dst, immI con)
15263 %{
15264   predicate(n->as_Vector()->length() == 16);
15265   match(Set dst (ReplicateB con));
15266   ins_cost(INSN_COST);
15267   format %{ "movi  $dst, $con\t# vector(16B)" %}
15268   ins_encode %{
15269     __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant & 0xff);
15270   %}
15271   ins_pipe(vmovi_reg_imm128);
15272 %}
15273 
15274 instruct replicate4S(vecD dst, iRegIorL2I src)
15275 %{
15276   predicate(n->as_Vector()->length() == 2 ||
15277             n->as_Vector()->length() == 4);
15278   match(Set dst (ReplicateS src));
15279   ins_cost(INSN_COST);
15280   format %{ "dup  $dst, $src\t# vector (4S)" %}
15281   ins_encode %{
15282     __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
15283   %}
15284   ins_pipe(vdup_reg_reg64);
15285 %}
15286 
15287 instruct replicate8S(vecX dst, iRegIorL2I src)
15288 %{
15289   predicate(n->as_Vector()->length() == 8);
15290   match(Set dst (ReplicateS src));
15291   ins_cost(INSN_COST);
15292   format %{ "dup  $dst, $src\t# vector (8S)" %}
15293   ins_encode %{
15294     __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
15295   %}
15296   ins_pipe(vdup_reg_reg128);
15297 %}
15298 
15299 instruct replicate4S_imm(vecD dst, immI con)
15300 %{
15301   predicate(n->as_Vector()->length() == 2 ||
15302             n->as_Vector()->length() == 4);
15303   match(Set dst (ReplicateS con));
15304   ins_cost(INSN_COST);
15305   format %{ "movi  $dst, $con\t# vector(4H)" %}
15306   ins_encode %{
15307     __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant & 0xffff);
15308   %}
15309   ins_pipe(vmovi_reg_imm64);
15310 %}
15311 
15312 instruct replicate8S_imm(vecX dst, immI con)
15313 %{
15314   predicate(n->as_Vector()->length() == 8);
15315   match(Set dst (ReplicateS con));
15316   ins_cost(INSN_COST);
15317   format %{ "movi  $dst, $con\t# vector(8H)" %}
15318   ins_encode %{
15319     __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant & 0xffff);
15320   %}
15321   ins_pipe(vmovi_reg_imm128);
15322 %}
15323 
15324 instruct replicate2I(vecD dst, iRegIorL2I src)
15325 %{
15326   predicate(n->as_Vector()->length() == 2);
15327   match(Set dst (ReplicateI src));
15328   ins_cost(INSN_COST);
15329   format %{ "dup  $dst, $src\t# vector (2I)" %}
15330   ins_encode %{
15331     __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
15332   %}
15333   ins_pipe(vdup_reg_reg64);
15334 %}
15335 
15336 instruct replicate4I(vecX dst, iRegIorL2I src)
15337 %{
15338   predicate(n->as_Vector()->length() == 4);
15339   match(Set dst (ReplicateI src));
15340   ins_cost(INSN_COST);
15341   format %{ "dup  $dst, $src\t# vector (4I)" %}
15342   ins_encode %{
15343     __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
15344   %}
15345   ins_pipe(vdup_reg_reg128);
15346 %}
15347 
15348 instruct replicate2I_imm(vecD dst, immI con)
15349 %{
15350   predicate(n->as_Vector()->length() == 2);
15351   match(Set dst (ReplicateI con));
15352   ins_cost(INSN_COST);
15353   format %{ "movi  $dst, $con\t# vector(2I)" %}
15354   ins_encode %{
15355     __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
15356   %}
15357   ins_pipe(vmovi_reg_imm64);
15358 %}
15359 
15360 instruct replicate4I_imm(vecX dst, immI con)
15361 %{
15362   predicate(n->as_Vector()->length() == 4);
15363   match(Set dst (ReplicateI con));
15364   ins_cost(INSN_COST);
15365   format %{ "movi  $dst, $con\t# vector(4I)" %}
15366   ins_encode %{
15367     __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);
15368   %}
15369   ins_pipe(vmovi_reg_imm128);
15370 %}
15371 
15372 instruct replicate2L(vecX dst, iRegL src)
15373 %{
15374   predicate(n->as_Vector()->length() == 2);
15375   match(Set dst (ReplicateL src));
15376   ins_cost(INSN_COST);
15377   format %{ "dup  $dst, $src\t# vector (2L)" %}
15378   ins_encode %{
15379     __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));
15380   %}
15381   ins_pipe(vdup_reg_reg128);
15382 %}
15383 
15384 instruct replicate2L_zero(vecX dst, immI0 zero)
15385 %{
15386   predicate(n->as_Vector()->length() == 2);
15387   match(Set dst (ReplicateI zero));
15388   ins_cost(INSN_COST);
15389   format %{ "movi  $dst, $zero\t# vector(4I)" %}
15390   ins_encode %{
15391     __ eor(as_FloatRegister($dst$$reg), __ T16B,
15392            as_FloatRegister($dst$$reg),
15393            as_FloatRegister($dst$$reg));
15394   %}
15395   ins_pipe(vmovi_reg_imm128);
15396 %}
15397 
15398 instruct replicate2F(vecD dst, vRegF src)
15399 %{
15400   predicate(n->as_Vector()->length() == 2);
15401   match(Set dst (ReplicateF src));
15402   ins_cost(INSN_COST);
15403   format %{ "dup  $dst, $src\t# vector (2F)" %}
15404   ins_encode %{
15405     __ dup(as_FloatRegister($dst$$reg), __ T2S,
15406            as_FloatRegister($src$$reg));
15407   %}
15408   ins_pipe(vdup_reg_freg64);
15409 %}
15410 
15411 instruct replicate4F(vecX dst, vRegF src)
15412 %{
15413   predicate(n->as_Vector()->length() == 4);
15414   match(Set dst (ReplicateF src));
15415   ins_cost(INSN_COST);
15416   format %{ "dup  $dst, $src\t# vector (4F)" %}
15417   ins_encode %{
15418     __ dup(as_FloatRegister($dst$$reg), __ T4S,
15419            as_FloatRegister($src$$reg));
15420   %}
15421   ins_pipe(vdup_reg_freg128);
15422 %}
15423 
15424 instruct replicate2D(vecX dst, vRegD src)
15425 %{
15426   predicate(n->as_Vector()->length() == 2);
15427   match(Set dst (ReplicateD src));
15428   ins_cost(INSN_COST);
15429   format %{ "dup  $dst, $src\t# vector (2D)" %}
15430   ins_encode %{
15431     __ dup(as_FloatRegister($dst$$reg), __ T2D,
15432            as_FloatRegister($src$$reg));
15433   %}
15434   ins_pipe(vdup_reg_dreg128);
15435 %}
15436 
15437 // ====================REDUCTION ARITHMETIC====================================
15438 
15439 instruct reduce_add2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp, iRegINoSp tmp2)
15440 %{
15441   match(Set dst (AddReductionVI src1 src2));
15442   ins_cost(INSN_COST);
15443   effect(TEMP tmp, TEMP tmp2);
15444   format %{ "umov  $tmp, $src2, S, 0\n\t"
15445             "umov  $tmp2, $src2, S, 1\n\t"
15446             "addw  $dst, $src1, $tmp\n\t"
15447             "addw  $dst, $dst, $tmp2\t add reduction2i"
15448   %}
15449   ins_encode %{
15450     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
15451     __ umov($tmp2$$Register, as_FloatRegister($src2$$reg), __ S, 1);
15452     __ addw($dst$$Register, $src1$$Register, $tmp$$Register);
15453     __ addw($dst$$Register, $dst$$Register, $tmp2$$Register);
15454   %}
15455   ins_pipe(pipe_class_default);
15456 %}
15457 
15458 instruct reduce_add4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
15459 %{
15460   match(Set dst (AddReductionVI src1 src2));
15461   ins_cost(INSN_COST);
15462   effect(TEMP tmp, TEMP tmp2);
15463   format %{ "addv  $tmp, T4S, $src2\n\t"
15464             "umov  $tmp2, $tmp, S, 0\n\t"
15465             "addw  $dst, $tmp2, $src1\t add reduction4i"
15466   %}
15467   ins_encode %{
15468     __ addv(as_FloatRegister($tmp$$reg), __ T4S,
15469             as_FloatRegister($src2$$reg));
15470     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
15471     __ addw($dst$$Register, $tmp2$$Register, $src1$$Register);
15472   %}
15473   ins_pipe(pipe_class_default);
15474 %}
15475 
15476 instruct reduce_mul2I(iRegINoSp dst, iRegIorL2I src1, vecD src2, iRegINoSp tmp)
15477 %{
15478   match(Set dst (MulReductionVI src1 src2));
15479   ins_cost(INSN_COST);
15480   effect(TEMP tmp, TEMP dst);
15481   format %{ "umov  $tmp, $src2, S, 0\n\t"
15482             "mul   $dst, $tmp, $src1\n\t"
15483             "umov  $tmp, $src2, S, 1\n\t"
15484             "mul   $dst, $tmp, $dst\t mul reduction2i\n\t"
15485   %}
15486   ins_encode %{
15487     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 0);
15488     __ mul($dst$$Register, $tmp$$Register, $src1$$Register);
15489     __ umov($tmp$$Register, as_FloatRegister($src2$$reg), __ S, 1);
15490     __ mul($dst$$Register, $tmp$$Register, $dst$$Register);
15491   %}
15492   ins_pipe(pipe_class_default);
15493 %}
15494 
15495 instruct reduce_mul4I(iRegINoSp dst, iRegIorL2I src1, vecX src2, vecX tmp, iRegINoSp tmp2)
15496 %{
15497   match(Set dst (MulReductionVI src1 src2));
15498   ins_cost(INSN_COST);
15499   effect(TEMP tmp, TEMP tmp2, TEMP dst);
15500   format %{ "ins   $tmp, $src2, 0, 1\n\t"
15501             "mul   $tmp, $tmp, $src2\n\t"
15502             "umov  $tmp2, $tmp, S, 0\n\t"
15503             "mul   $dst, $tmp2, $src1\n\t"
15504             "umov  $tmp2, $tmp, S, 1\n\t"
15505             "mul   $dst, $tmp2, $dst\t mul reduction4i\n\t"
15506   %}
15507   ins_encode %{
15508     __ ins(as_FloatRegister($tmp$$reg), __ D,
15509            as_FloatRegister($src2$$reg), 0, 1);
15510     __ mulv(as_FloatRegister($tmp$$reg), __ T2S,
15511            as_FloatRegister($tmp$$reg), as_FloatRegister($src2$$reg));
15512     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
15513     __ mul($dst$$Register, $tmp2$$Register, $src1$$Register);
15514     __ umov($tmp2$$Register, as_FloatRegister($tmp$$reg), __ S, 1);
15515     __ mul($dst$$Register, $tmp2$$Register, $dst$$Register);
15516   %}
15517   ins_pipe(pipe_class_default);
15518 %}
15519 
15520 instruct reduce_add2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
15521 %{
15522   match(Set dst (AddReductionVF src1 src2));
15523   ins_cost(INSN_COST);
15524   effect(TEMP tmp, TEMP dst);
15525   format %{ "fadds $dst, $src1, $src2\n\t"
15526             "ins   $tmp, S, $src2, 0, 1\n\t"
15527             "fadds $dst, $dst, $tmp\t add reduction2f"
15528   %}
15529   ins_encode %{
15530     __ fadds(as_FloatRegister($dst$$reg),
15531              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15532     __ ins(as_FloatRegister($tmp$$reg), __ S,
15533            as_FloatRegister($src2$$reg), 0, 1);
15534     __ fadds(as_FloatRegister($dst$$reg),
15535              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15536   %}
15537   ins_pipe(pipe_class_default);
15538 %}
15539 
15540 instruct reduce_add4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
15541 %{
15542   match(Set dst (AddReductionVF src1 src2));
15543   ins_cost(INSN_COST);
15544   effect(TEMP tmp, TEMP dst);
15545   format %{ "fadds $dst, $src1, $src2\n\t"
15546             "ins   $tmp, S, $src2, 0, 1\n\t"
15547             "fadds $dst, $dst, $tmp\n\t"
15548             "ins   $tmp, S, $src2, 0, 2\n\t"
15549             "fadds $dst, $dst, $tmp\n\t"
15550             "ins   $tmp, S, $src2, 0, 3\n\t"
15551             "fadds $dst, $dst, $tmp\t add reduction4f"
15552   %}
15553   ins_encode %{
15554     __ fadds(as_FloatRegister($dst$$reg),
15555              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15556     __ ins(as_FloatRegister($tmp$$reg), __ S,
15557            as_FloatRegister($src2$$reg), 0, 1);
15558     __ fadds(as_FloatRegister($dst$$reg),
15559              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15560     __ ins(as_FloatRegister($tmp$$reg), __ S,
15561            as_FloatRegister($src2$$reg), 0, 2);
15562     __ fadds(as_FloatRegister($dst$$reg),
15563              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15564     __ ins(as_FloatRegister($tmp$$reg), __ S,
15565            as_FloatRegister($src2$$reg), 0, 3);
15566     __ fadds(as_FloatRegister($dst$$reg),
15567              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15568   %}
15569   ins_pipe(pipe_class_default);
15570 %}
15571 
15572 instruct reduce_mul2F(vRegF dst, vRegF src1, vecD src2, vecD tmp)
15573 %{
15574   match(Set dst (MulReductionVF src1 src2));
15575   ins_cost(INSN_COST);
15576   effect(TEMP tmp, TEMP dst);
15577   format %{ "fmuls $dst, $src1, $src2\n\t"
15578             "ins   $tmp, S, $src2, 0, 1\n\t"
15579             "fmuls $dst, $dst, $tmp\t add reduction4f"
15580   %}
15581   ins_encode %{
15582     __ fmuls(as_FloatRegister($dst$$reg),
15583              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15584     __ ins(as_FloatRegister($tmp$$reg), __ S,
15585            as_FloatRegister($src2$$reg), 0, 1);
15586     __ fmuls(as_FloatRegister($dst$$reg),
15587              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15588   %}
15589   ins_pipe(pipe_class_default);
15590 %}
15591 
15592 instruct reduce_mul4F(vRegF dst, vRegF src1, vecX src2, vecX tmp)
15593 %{
15594   match(Set dst (MulReductionVF src1 src2));
15595   ins_cost(INSN_COST);
15596   effect(TEMP tmp, TEMP dst);
15597   format %{ "fmuls $dst, $src1, $src2\n\t"
15598             "ins   $tmp, S, $src2, 0, 1\n\t"
15599             "fmuls $dst, $dst, $tmp\n\t"
15600             "ins   $tmp, S, $src2, 0, 2\n\t"
15601             "fmuls $dst, $dst, $tmp\n\t"
15602             "ins   $tmp, S, $src2, 0, 3\n\t"
15603             "fmuls $dst, $dst, $tmp\t add reduction4f"
15604   %}
15605   ins_encode %{
15606     __ fmuls(as_FloatRegister($dst$$reg),
15607              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15608     __ ins(as_FloatRegister($tmp$$reg), __ S,
15609            as_FloatRegister($src2$$reg), 0, 1);
15610     __ fmuls(as_FloatRegister($dst$$reg),
15611              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15612     __ ins(as_FloatRegister($tmp$$reg), __ S,
15613            as_FloatRegister($src2$$reg), 0, 2);
15614     __ fmuls(as_FloatRegister($dst$$reg),
15615              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15616     __ ins(as_FloatRegister($tmp$$reg), __ S,
15617            as_FloatRegister($src2$$reg), 0, 3);
15618     __ fmuls(as_FloatRegister($dst$$reg),
15619              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15620   %}
15621   ins_pipe(pipe_class_default);
15622 %}
15623 
15624 instruct reduce_add2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
15625 %{
15626   match(Set dst (AddReductionVD src1 src2));
15627   ins_cost(INSN_COST);
15628   effect(TEMP tmp, TEMP dst);
15629   format %{ "faddd $dst, $src1, $src2\n\t"
15630             "ins   $tmp, D, $src2, 0, 1\n\t"
15631             "faddd $dst, $dst, $tmp\t add reduction2d"
15632   %}
15633   ins_encode %{
15634     __ faddd(as_FloatRegister($dst$$reg),
15635              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15636     __ ins(as_FloatRegister($tmp$$reg), __ D,
15637            as_FloatRegister($src2$$reg), 0, 1);
15638     __ faddd(as_FloatRegister($dst$$reg),
15639              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15640   %}
15641   ins_pipe(pipe_class_default);
15642 %}
15643 
15644 instruct reduce_mul2D(vRegD dst, vRegD src1, vecX src2, vecX tmp)
15645 %{
15646   match(Set dst (MulReductionVD src1 src2));
15647   ins_cost(INSN_COST);
15648   effect(TEMP tmp, TEMP dst);
15649   format %{ "fmuld $dst, $src1, $src2\n\t"
15650             "ins   $tmp, D, $src2, 0, 1\n\t"
15651             "fmuld $dst, $dst, $tmp\t add reduction2d"
15652   %}
15653   ins_encode %{
15654     __ fmuld(as_FloatRegister($dst$$reg),
15655              as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15656     __ ins(as_FloatRegister($tmp$$reg), __ D,
15657            as_FloatRegister($src2$$reg), 0, 1);
15658     __ fmuld(as_FloatRegister($dst$$reg),
15659              as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15660   %}
15661   ins_pipe(pipe_class_default);
15662 %}
15663 
15664 instruct reduce_max2F(vRegF dst, vRegF src1, vecD src2, vecD tmp) %{
15665   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15666   match(Set dst (MaxReductionV src1 src2));
15667   ins_cost(INSN_COST);
15668   effect(TEMP_DEF dst, TEMP tmp);
15669   format %{ "fmaxs $dst, $src1, $src2\n\t"
15670             "ins   $tmp, S, $src2, 0, 1\n\t"
15671             "fmaxs $dst, $dst, $tmp\t max reduction2F" %}
15672   ins_encode %{
15673     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15674     __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($src2$$reg), 0, 1);
15675     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15676   %}
15677   ins_pipe(pipe_class_default);
15678 %}
15679 
15680 instruct reduce_max4F(vRegF dst, vRegF src1, vecX src2) %{
15681   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15682   match(Set dst (MaxReductionV src1 src2));
15683   ins_cost(INSN_COST);
15684   effect(TEMP_DEF dst);
15685   format %{ "fmaxv $dst, T4S, $src2\n\t"
15686             "fmaxs $dst, $dst, $src1\t max reduction4F" %}
15687   ins_encode %{
15688     __ fmaxv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src2$$reg));
15689     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
15690   %}
15691   ins_pipe(pipe_class_default);
15692 %}
15693 
15694 instruct reduce_max2D(vRegD dst, vRegD src1, vecX src2, vecX tmp) %{
15695   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15696   match(Set dst (MaxReductionV src1 src2));
15697   ins_cost(INSN_COST);
15698   effect(TEMP_DEF dst, TEMP tmp);
15699   format %{ "fmaxd $dst, $src1, $src2\n\t"
15700             "ins   $tmp, D, $src2, 0, 1\n\t"
15701             "fmaxd $dst, $dst, $tmp\t max reduction2D" %}
15702   ins_encode %{
15703     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15704     __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($src2$$reg), 0, 1);
15705     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15706   %}
15707   ins_pipe(pipe_class_default);
15708 %}
15709 
15710 instruct reduce_min2F(vRegF dst, vRegF src1, vecD src2, vecD tmp) %{
15711   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15712   match(Set dst (MinReductionV src1 src2));
15713   ins_cost(INSN_COST);
15714   effect(TEMP_DEF dst, TEMP tmp);
15715   format %{ "fmins $dst, $src1, $src2\n\t"
15716             "ins   $tmp, S, $src2, 0, 1\n\t"
15717             "fmins $dst, $dst, $tmp\t min reduction2F" %}
15718   ins_encode %{
15719     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15720     __ ins(as_FloatRegister($tmp$$reg), __ S, as_FloatRegister($src2$$reg), 0, 1);
15721     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15722   %}
15723   ins_pipe(pipe_class_default);
15724 %}
15725 
15726 instruct reduce_min4F(vRegF dst, vRegF src1, vecX src2) %{
15727   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15728   match(Set dst (MinReductionV src1 src2));
15729   ins_cost(INSN_COST);
15730   effect(TEMP_DEF dst);
15731   format %{ "fminv $dst, T4S, $src2\n\t"
15732             "fmins $dst, $dst, $src1\t min reduction4F" %}
15733   ins_encode %{
15734     __ fminv(as_FloatRegister($dst$$reg), __ T4S, as_FloatRegister($src2$$reg));
15735     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
15736   %}
15737   ins_pipe(pipe_class_default);
15738 %}
15739 
15740 instruct reduce_min2D(vRegD dst, vRegD src1, vecX src2, vecX tmp) %{
15741   predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15742   match(Set dst (MinReductionV src1 src2));
15743   ins_cost(INSN_COST);
15744   effect(TEMP_DEF dst, TEMP tmp);
15745   format %{ "fmind $dst, $src1, $src2\n\t"
15746             "ins   $tmp, D, $src2, 0, 1\n\t"
15747             "fmind $dst, $dst, $tmp\t min reduction2D" %}
15748   ins_encode %{
15749     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15750     __ ins(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($src2$$reg), 0, 1);
15751     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
15752   %}
15753   ins_pipe(pipe_class_default);
15754 %}
15755 
15756 // ====================VECTOR ARITHMETIC=======================================
15757 
15758 // --------------------------------- ADD --------------------------------------
15759 
15760 instruct vadd8B(vecD dst, vecD src1, vecD src2)
15761 %{
15762   predicate(n->as_Vector()->length() == 4 ||
15763             n->as_Vector()->length() == 8);
15764   match(Set dst (AddVB src1 src2));
15765   ins_cost(INSN_COST);
15766   format %{ "addv  $dst,$src1,$src2\t# vector (8B)" %}
15767   ins_encode %{
15768     __ addv(as_FloatRegister($dst$$reg), __ T8B,
15769             as_FloatRegister($src1$$reg),
15770             as_FloatRegister($src2$$reg));
15771   %}
15772   ins_pipe(vdop64);
15773 %}
15774 
15775 instruct vadd16B(vecX dst, vecX src1, vecX src2)
15776 %{
15777   predicate(n->as_Vector()->length() == 16);
15778   match(Set dst (AddVB src1 src2));
15779   ins_cost(INSN_COST);
15780   format %{ "addv  $dst,$src1,$src2\t# vector (16B)" %}
15781   ins_encode %{
15782     __ addv(as_FloatRegister($dst$$reg), __ T16B,
15783             as_FloatRegister($src1$$reg),
15784             as_FloatRegister($src2$$reg));
15785   %}
15786   ins_pipe(vdop128);
15787 %}
15788 
15789 instruct vadd4S(vecD dst, vecD src1, vecD src2)
15790 %{
15791   predicate(n->as_Vector()->length() == 2 ||
15792             n->as_Vector()->length() == 4);
15793   match(Set dst (AddVS src1 src2));
15794   ins_cost(INSN_COST);
15795   format %{ "addv  $dst,$src1,$src2\t# vector (4H)" %}
15796   ins_encode %{
15797     __ addv(as_FloatRegister($dst$$reg), __ T4H,
15798             as_FloatRegister($src1$$reg),
15799             as_FloatRegister($src2$$reg));
15800   %}
15801   ins_pipe(vdop64);
15802 %}
15803 
15804 instruct vadd8S(vecX dst, vecX src1, vecX src2)
15805 %{
15806   predicate(n->as_Vector()->length() == 8);
15807   match(Set dst (AddVS src1 src2));
15808   ins_cost(INSN_COST);
15809   format %{ "addv  $dst,$src1,$src2\t# vector (8H)" %}
15810   ins_encode %{
15811     __ addv(as_FloatRegister($dst$$reg), __ T8H,
15812             as_FloatRegister($src1$$reg),
15813             as_FloatRegister($src2$$reg));
15814   %}
15815   ins_pipe(vdop128);
15816 %}
15817 
15818 instruct vadd2I(vecD dst, vecD src1, vecD src2)
15819 %{
15820   predicate(n->as_Vector()->length() == 2);
15821   match(Set dst (AddVI src1 src2));
15822   ins_cost(INSN_COST);
15823   format %{ "addv  $dst,$src1,$src2\t# vector (2S)" %}
15824   ins_encode %{
15825     __ addv(as_FloatRegister($dst$$reg), __ T2S,
15826             as_FloatRegister($src1$$reg),
15827             as_FloatRegister($src2$$reg));
15828   %}
15829   ins_pipe(vdop64);
15830 %}
15831 
15832 instruct vadd4I(vecX dst, vecX src1, vecX src2)
15833 %{
15834   predicate(n->as_Vector()->length() == 4);
15835   match(Set dst (AddVI src1 src2));
15836   ins_cost(INSN_COST);
15837   format %{ "addv  $dst,$src1,$src2\t# vector (4S)" %}
15838   ins_encode %{
15839     __ addv(as_FloatRegister($dst$$reg), __ T4S,
15840             as_FloatRegister($src1$$reg),
15841             as_FloatRegister($src2$$reg));
15842   %}
15843   ins_pipe(vdop128);
15844 %}
15845 
15846 instruct vadd2L(vecX dst, vecX src1, vecX src2)
15847 %{
15848   predicate(n->as_Vector()->length() == 2);
15849   match(Set dst (AddVL src1 src2));
15850   ins_cost(INSN_COST);
15851   format %{ "addv  $dst,$src1,$src2\t# vector (2L)" %}
15852   ins_encode %{
15853     __ addv(as_FloatRegister($dst$$reg), __ T2D,
15854             as_FloatRegister($src1$$reg),
15855             as_FloatRegister($src2$$reg));
15856   %}
15857   ins_pipe(vdop128);
15858 %}
15859 
15860 instruct vadd2F(vecD dst, vecD src1, vecD src2)
15861 %{
15862   predicate(n->as_Vector()->length() == 2);
15863   match(Set dst (AddVF src1 src2));
15864   ins_cost(INSN_COST);
15865   format %{ "fadd  $dst,$src1,$src2\t# vector (2S)" %}
15866   ins_encode %{
15867     __ fadd(as_FloatRegister($dst$$reg), __ T2S,
15868             as_FloatRegister($src1$$reg),
15869             as_FloatRegister($src2$$reg));
15870   %}
15871   ins_pipe(vdop_fp64);
15872 %}
15873 
15874 instruct vadd4F(vecX dst, vecX src1, vecX src2)
15875 %{
15876   predicate(n->as_Vector()->length() == 4);
15877   match(Set dst (AddVF src1 src2));
15878   ins_cost(INSN_COST);
15879   format %{ "fadd  $dst,$src1,$src2\t# vector (4S)" %}
15880   ins_encode %{
15881     __ fadd(as_FloatRegister($dst$$reg), __ T4S,
15882             as_FloatRegister($src1$$reg),
15883             as_FloatRegister($src2$$reg));
15884   %}
15885   ins_pipe(vdop_fp128);
15886 %}
15887 
15888 instruct vadd2D(vecX dst, vecX src1, vecX src2)
15889 %{
15890   match(Set dst (AddVD src1 src2));
15891   ins_cost(INSN_COST);
15892   format %{ "fadd  $dst,$src1,$src2\t# vector (2D)" %}
15893   ins_encode %{
15894     __ fadd(as_FloatRegister($dst$$reg), __ T2D,
15895             as_FloatRegister($src1$$reg),
15896             as_FloatRegister($src2$$reg));
15897   %}
15898   ins_pipe(vdop_fp128);
15899 %}
15900 
15901 // --------------------------------- SUB --------------------------------------
15902 
15903 instruct vsub8B(vecD dst, vecD src1, vecD src2)
15904 %{
15905   predicate(n->as_Vector()->length() == 4 ||
15906             n->as_Vector()->length() == 8);
15907   match(Set dst (SubVB src1 src2));
15908   ins_cost(INSN_COST);
15909   format %{ "subv  $dst,$src1,$src2\t# vector (8B)" %}
15910   ins_encode %{
15911     __ subv(as_FloatRegister($dst$$reg), __ T8B,
15912             as_FloatRegister($src1$$reg),
15913             as_FloatRegister($src2$$reg));
15914   %}
15915   ins_pipe(vdop64);
15916 %}
15917 
15918 instruct vsub16B(vecX dst, vecX src1, vecX src2)
15919 %{
15920   predicate(n->as_Vector()->length() == 16);
15921   match(Set dst (SubVB src1 src2));
15922   ins_cost(INSN_COST);
15923   format %{ "subv  $dst,$src1,$src2\t# vector (16B)" %}
15924   ins_encode %{
15925     __ subv(as_FloatRegister($dst$$reg), __ T16B,
15926             as_FloatRegister($src1$$reg),
15927             as_FloatRegister($src2$$reg));
15928   %}
15929   ins_pipe(vdop128);
15930 %}
15931 
15932 instruct vsub4S(vecD dst, vecD src1, vecD src2)
15933 %{
15934   predicate(n->as_Vector()->length() == 2 ||
15935             n->as_Vector()->length() == 4);
15936   match(Set dst (SubVS src1 src2));
15937   ins_cost(INSN_COST);
15938   format %{ "subv  $dst,$src1,$src2\t# vector (4H)" %}
15939   ins_encode %{
15940     __ subv(as_FloatRegister($dst$$reg), __ T4H,
15941             as_FloatRegister($src1$$reg),
15942             as_FloatRegister($src2$$reg));
15943   %}
15944   ins_pipe(vdop64);
15945 %}
15946 
15947 instruct vsub8S(vecX dst, vecX src1, vecX src2)
15948 %{
15949   predicate(n->as_Vector()->length() == 8);
15950   match(Set dst (SubVS src1 src2));
15951   ins_cost(INSN_COST);
15952   format %{ "subv  $dst,$src1,$src2\t# vector (8H)" %}
15953   ins_encode %{
15954     __ subv(as_FloatRegister($dst$$reg), __ T8H,
15955             as_FloatRegister($src1$$reg),
15956             as_FloatRegister($src2$$reg));
15957   %}
15958   ins_pipe(vdop128);
15959 %}
15960 
15961 instruct vsub2I(vecD dst, vecD src1, vecD src2)
15962 %{
15963   predicate(n->as_Vector()->length() == 2);
15964   match(Set dst (SubVI src1 src2));
15965   ins_cost(INSN_COST);
15966   format %{ "subv  $dst,$src1,$src2\t# vector (2S)" %}
15967   ins_encode %{
15968     __ subv(as_FloatRegister($dst$$reg), __ T2S,
15969             as_FloatRegister($src1$$reg),
15970             as_FloatRegister($src2$$reg));
15971   %}
15972   ins_pipe(vdop64);
15973 %}
15974 
15975 instruct vsub4I(vecX dst, vecX src1, vecX src2)
15976 %{
15977   predicate(n->as_Vector()->length() == 4);
15978   match(Set dst (SubVI src1 src2));
15979   ins_cost(INSN_COST);
15980   format %{ "subv  $dst,$src1,$src2\t# vector (4S)" %}
15981   ins_encode %{
15982     __ subv(as_FloatRegister($dst$$reg), __ T4S,
15983             as_FloatRegister($src1$$reg),
15984             as_FloatRegister($src2$$reg));
15985   %}
15986   ins_pipe(vdop128);
15987 %}
15988 
15989 instruct vsub2L(vecX dst, vecX src1, vecX src2)
15990 %{
15991   predicate(n->as_Vector()->length() == 2);
15992   match(Set dst (SubVL src1 src2));
15993   ins_cost(INSN_COST);
15994   format %{ "subv  $dst,$src1,$src2\t# vector (2L)" %}
15995   ins_encode %{
15996     __ subv(as_FloatRegister($dst$$reg), __ T2D,
15997             as_FloatRegister($src1$$reg),
15998             as_FloatRegister($src2$$reg));
15999   %}
16000   ins_pipe(vdop128);
16001 %}
16002 
16003 instruct vsub2F(vecD dst, vecD src1, vecD src2)
16004 %{
16005   predicate(n->as_Vector()->length() == 2);
16006   match(Set dst (SubVF src1 src2));
16007   ins_cost(INSN_COST);
16008   format %{ "fsub  $dst,$src1,$src2\t# vector (2S)" %}
16009   ins_encode %{
16010     __ fsub(as_FloatRegister($dst$$reg), __ T2S,
16011             as_FloatRegister($src1$$reg),
16012             as_FloatRegister($src2$$reg));
16013   %}
16014   ins_pipe(vdop_fp64);
16015 %}
16016 
16017 instruct vsub4F(vecX dst, vecX src1, vecX src2)
16018 %{
16019   predicate(n->as_Vector()->length() == 4);
16020   match(Set dst (SubVF src1 src2));
16021   ins_cost(INSN_COST);
16022   format %{ "fsub  $dst,$src1,$src2\t# vector (4S)" %}
16023   ins_encode %{
16024     __ fsub(as_FloatRegister($dst$$reg), __ T4S,
16025             as_FloatRegister($src1$$reg),
16026             as_FloatRegister($src2$$reg));
16027   %}
16028   ins_pipe(vdop_fp128);
16029 %}
16030 
16031 instruct vsub2D(vecX dst, vecX src1, vecX src2)
16032 %{
16033   predicate(n->as_Vector()->length() == 2);
16034   match(Set dst (SubVD src1 src2));
16035   ins_cost(INSN_COST);
16036   format %{ "fsub  $dst,$src1,$src2\t# vector (2D)" %}
16037   ins_encode %{
16038     __ fsub(as_FloatRegister($dst$$reg), __ T2D,
16039             as_FloatRegister($src1$$reg),
16040             as_FloatRegister($src2$$reg));
16041   %}
16042   ins_pipe(vdop_fp128);
16043 %}
16044 
16045 // --------------------------------- MUL --------------------------------------
16046 
16047 instruct vmul4S(vecD dst, vecD src1, vecD src2)
16048 %{
16049   predicate(n->as_Vector()->length() == 2 ||
16050             n->as_Vector()->length() == 4);
16051   match(Set dst (MulVS src1 src2));
16052   ins_cost(INSN_COST);
16053   format %{ "mulv  $dst,$src1,$src2\t# vector (4H)" %}
16054   ins_encode %{
16055     __ mulv(as_FloatRegister($dst$$reg), __ T4H,
16056             as_FloatRegister($src1$$reg),
16057             as_FloatRegister($src2$$reg));
16058   %}
16059   ins_pipe(vmul64);
16060 %}
16061 
16062 instruct vmul8S(vecX dst, vecX src1, vecX src2)
16063 %{
16064   predicate(n->as_Vector()->length() == 8);
16065   match(Set dst (MulVS src1 src2));
16066   ins_cost(INSN_COST);
16067   format %{ "mulv  $dst,$src1,$src2\t# vector (8H)" %}
16068   ins_encode %{
16069     __ mulv(as_FloatRegister($dst$$reg), __ T8H,
16070             as_FloatRegister($src1$$reg),
16071             as_FloatRegister($src2$$reg));
16072   %}
16073   ins_pipe(vmul128);
16074 %}
16075 
16076 instruct vmul2I(vecD dst, vecD src1, vecD src2)
16077 %{
16078   predicate(n->as_Vector()->length() == 2);
16079   match(Set dst (MulVI src1 src2));
16080   ins_cost(INSN_COST);
16081   format %{ "mulv  $dst,$src1,$src2\t# vector (2S)" %}
16082   ins_encode %{
16083     __ mulv(as_FloatRegister($dst$$reg), __ T2S,
16084             as_FloatRegister($src1$$reg),
16085             as_FloatRegister($src2$$reg));
16086   %}
16087   ins_pipe(vmul64);
16088 %}
16089 
16090 instruct vmul4I(vecX dst, vecX src1, vecX src2)
16091 %{
16092   predicate(n->as_Vector()->length() == 4);
16093   match(Set dst (MulVI src1 src2));
16094   ins_cost(INSN_COST);
16095   format %{ "mulv  $dst,$src1,$src2\t# vector (4S)" %}
16096   ins_encode %{
16097     __ mulv(as_FloatRegister($dst$$reg), __ T4S,
16098             as_FloatRegister($src1$$reg),
16099             as_FloatRegister($src2$$reg));
16100   %}
16101   ins_pipe(vmul128);
16102 %}
16103 
16104 instruct vmul2F(vecD dst, vecD src1, vecD src2)
16105 %{
16106   predicate(n->as_Vector()->length() == 2);
16107   match(Set dst (MulVF src1 src2));
16108   ins_cost(INSN_COST);
16109   format %{ "fmul  $dst,$src1,$src2\t# vector (2S)" %}
16110   ins_encode %{
16111     __ fmul(as_FloatRegister($dst$$reg), __ T2S,
16112             as_FloatRegister($src1$$reg),
16113             as_FloatRegister($src2$$reg));
16114   %}
16115   ins_pipe(vmuldiv_fp64);
16116 %}
16117 
16118 instruct vmul4F(vecX dst, vecX src1, vecX src2)
16119 %{
16120   predicate(n->as_Vector()->length() == 4);
16121   match(Set dst (MulVF src1 src2));
16122   ins_cost(INSN_COST);
16123   format %{ "fmul  $dst,$src1,$src2\t# vector (4S)" %}
16124   ins_encode %{
16125     __ fmul(as_FloatRegister($dst$$reg), __ T4S,
16126             as_FloatRegister($src1$$reg),
16127             as_FloatRegister($src2$$reg));
16128   %}
16129   ins_pipe(vmuldiv_fp128);
16130 %}
16131 
16132 instruct vmul2D(vecX dst, vecX src1, vecX src2)
16133 %{
16134   predicate(n->as_Vector()->length() == 2);
16135   match(Set dst (MulVD src1 src2));
16136   ins_cost(INSN_COST);
16137   format %{ "fmul  $dst,$src1,$src2\t# vector (2D)" %}
16138   ins_encode %{
16139     __ fmul(as_FloatRegister($dst$$reg), __ T2D,
16140             as_FloatRegister($src1$$reg),
16141             as_FloatRegister($src2$$reg));
16142   %}
16143   ins_pipe(vmuldiv_fp128);
16144 %}
16145 
16146 // --------------------------------- MLA --------------------------------------
16147 
16148 instruct vmla4S(vecD dst, vecD src1, vecD src2)
16149 %{
16150   predicate(n->as_Vector()->length() == 2 ||
16151             n->as_Vector()->length() == 4);
16152   match(Set dst (AddVS dst (MulVS src1 src2)));
16153   ins_cost(INSN_COST);
16154   format %{ "mlav  $dst,$src1,$src2\t# vector (4H)" %}
16155   ins_encode %{
16156     __ mlav(as_FloatRegister($dst$$reg), __ T4H,
16157             as_FloatRegister($src1$$reg),
16158             as_FloatRegister($src2$$reg));
16159   %}
16160   ins_pipe(vmla64);
16161 %}
16162 
16163 instruct vmla8S(vecX dst, vecX src1, vecX src2)
16164 %{
16165   predicate(n->as_Vector()->length() == 8);
16166   match(Set dst (AddVS dst (MulVS src1 src2)));
16167   ins_cost(INSN_COST);
16168   format %{ "mlav  $dst,$src1,$src2\t# vector (8H)" %}
16169   ins_encode %{
16170     __ mlav(as_FloatRegister($dst$$reg), __ T8H,
16171             as_FloatRegister($src1$$reg),
16172             as_FloatRegister($src2$$reg));
16173   %}
16174   ins_pipe(vmla128);
16175 %}
16176 
16177 instruct vmla2I(vecD dst, vecD src1, vecD src2)
16178 %{
16179   predicate(n->as_Vector()->length() == 2);
16180   match(Set dst (AddVI dst (MulVI src1 src2)));
16181   ins_cost(INSN_COST);
16182   format %{ "mlav  $dst,$src1,$src2\t# vector (2S)" %}
16183   ins_encode %{
16184     __ mlav(as_FloatRegister($dst$$reg), __ T2S,
16185             as_FloatRegister($src1$$reg),
16186             as_FloatRegister($src2$$reg));
16187   %}
16188   ins_pipe(vmla64);
16189 %}
16190 
16191 instruct vmla4I(vecX dst, vecX src1, vecX src2)
16192 %{
16193   predicate(n->as_Vector()->length() == 4);
16194   match(Set dst (AddVI dst (MulVI src1 src2)));
16195   ins_cost(INSN_COST);
16196   format %{ "mlav  $dst,$src1,$src2\t# vector (4S)" %}
16197   ins_encode %{
16198     __ mlav(as_FloatRegister($dst$$reg), __ T4S,
16199             as_FloatRegister($src1$$reg),
16200             as_FloatRegister($src2$$reg));
16201   %}
16202   ins_pipe(vmla128);
16203 %}
16204 
16205 // dst + src1 * src2
16206 instruct vmla2F(vecD dst, vecD src1, vecD src2) %{
16207   predicate(UseFMA && n->as_Vector()->length() == 2);
16208   match(Set dst (FmaVF  dst (Binary src1 src2)));
16209   format %{ "fmla  $dst,$src1,$src2\t# vector (2S)" %}
16210   ins_cost(INSN_COST);
16211   ins_encode %{
16212     __ fmla(as_FloatRegister($dst$$reg), __ T2S,
16213             as_FloatRegister($src1$$reg),
16214             as_FloatRegister($src2$$reg));
16215   %}
16216   ins_pipe(vmuldiv_fp64);
16217 %}
16218 
16219 // dst + src1 * src2
16220 instruct vmla4F(vecX dst, vecX src1, vecX src2) %{
16221   predicate(UseFMA && n->as_Vector()->length() == 4);
16222   match(Set dst (FmaVF  dst (Binary src1 src2)));
16223   format %{ "fmla  $dst,$src1,$src2\t# vector (4S)" %}
16224   ins_cost(INSN_COST);
16225   ins_encode %{
16226     __ fmla(as_FloatRegister($dst$$reg), __ T4S,
16227             as_FloatRegister($src1$$reg),
16228             as_FloatRegister($src2$$reg));
16229   %}
16230   ins_pipe(vmuldiv_fp128);
16231 %}
16232 
16233 // dst + src1 * src2
16234 instruct vmla2D(vecX dst, vecX src1, vecX src2) %{
16235   predicate(UseFMA && n->as_Vector()->length() == 2);
16236   match(Set dst (FmaVD  dst (Binary src1 src2)));
16237   format %{ "fmla  $dst,$src1,$src2\t# vector (2D)" %}
16238   ins_cost(INSN_COST);
16239   ins_encode %{
16240     __ fmla(as_FloatRegister($dst$$reg), __ T2D,
16241             as_FloatRegister($src1$$reg),
16242             as_FloatRegister($src2$$reg));
16243   %}
16244   ins_pipe(vmuldiv_fp128);
16245 %}
16246 
16247 // --------------------------------- MLS --------------------------------------
16248 
16249 instruct vmls4S(vecD dst, vecD src1, vecD src2)
16250 %{
16251   predicate(n->as_Vector()->length() == 2 ||
16252             n->as_Vector()->length() == 4);
16253   match(Set dst (SubVS dst (MulVS src1 src2)));
16254   ins_cost(INSN_COST);
16255   format %{ "mlsv  $dst,$src1,$src2\t# vector (4H)" %}
16256   ins_encode %{
16257     __ mlsv(as_FloatRegister($dst$$reg), __ T4H,
16258             as_FloatRegister($src1$$reg),
16259             as_FloatRegister($src2$$reg));
16260   %}
16261   ins_pipe(vmla64);
16262 %}
16263 
16264 instruct vmls8S(vecX dst, vecX src1, vecX src2)
16265 %{
16266   predicate(n->as_Vector()->length() == 8);
16267   match(Set dst (SubVS dst (MulVS src1 src2)));
16268   ins_cost(INSN_COST);
16269   format %{ "mlsv  $dst,$src1,$src2\t# vector (8H)" %}
16270   ins_encode %{
16271     __ mlsv(as_FloatRegister($dst$$reg), __ T8H,
16272             as_FloatRegister($src1$$reg),
16273             as_FloatRegister($src2$$reg));
16274   %}
16275   ins_pipe(vmla128);
16276 %}
16277 
16278 instruct vmls2I(vecD dst, vecD src1, vecD src2)
16279 %{
16280   predicate(n->as_Vector()->length() == 2);
16281   match(Set dst (SubVI dst (MulVI src1 src2)));
16282   ins_cost(INSN_COST);
16283   format %{ "mlsv  $dst,$src1,$src2\t# vector (2S)" %}
16284   ins_encode %{
16285     __ mlsv(as_FloatRegister($dst$$reg), __ T2S,
16286             as_FloatRegister($src1$$reg),
16287             as_FloatRegister($src2$$reg));
16288   %}
16289   ins_pipe(vmla64);
16290 %}
16291 
16292 instruct vmls4I(vecX dst, vecX src1, vecX src2)
16293 %{
16294   predicate(n->as_Vector()->length() == 4);
16295   match(Set dst (SubVI dst (MulVI src1 src2)));
16296   ins_cost(INSN_COST);
16297   format %{ "mlsv  $dst,$src1,$src2\t# vector (4S)" %}
16298   ins_encode %{
16299     __ mlsv(as_FloatRegister($dst$$reg), __ T4S,
16300             as_FloatRegister($src1$$reg),
16301             as_FloatRegister($src2$$reg));
16302   %}
16303   ins_pipe(vmla128);
16304 %}
16305 
16306 // dst - src1 * src2
16307 instruct vmls2F(vecD dst, vecD src1, vecD src2) %{
16308   predicate(UseFMA && n->as_Vector()->length() == 2);
16309   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
16310   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
16311   format %{ "fmls  $dst,$src1,$src2\t# vector (2S)" %}
16312   ins_cost(INSN_COST);
16313   ins_encode %{
16314     __ fmls(as_FloatRegister($dst$$reg), __ T2S,
16315             as_FloatRegister($src1$$reg),
16316             as_FloatRegister($src2$$reg));
16317   %}
16318   ins_pipe(vmuldiv_fp64);
16319 %}
16320 
16321 // dst - src1 * src2
16322 instruct vmls4F(vecX dst, vecX src1, vecX src2) %{
16323   predicate(UseFMA && n->as_Vector()->length() == 4);
16324   match(Set dst (FmaVF  dst (Binary (NegVF src1) src2)));
16325   match(Set dst (FmaVF  dst (Binary src1 (NegVF src2))));
16326   format %{ "fmls  $dst,$src1,$src2\t# vector (4S)" %}
16327   ins_cost(INSN_COST);
16328   ins_encode %{
16329     __ fmls(as_FloatRegister($dst$$reg), __ T4S,
16330             as_FloatRegister($src1$$reg),
16331             as_FloatRegister($src2$$reg));
16332   %}
16333   ins_pipe(vmuldiv_fp128);
16334 %}
16335 
16336 // dst - src1 * src2
16337 instruct vmls2D(vecX dst, vecX src1, vecX src2) %{
16338   predicate(UseFMA && n->as_Vector()->length() == 2);
16339   match(Set dst (FmaVD  dst (Binary (NegVD src1) src2)));
16340   match(Set dst (FmaVD  dst (Binary src1 (NegVD src2))));
16341   format %{ "fmls  $dst,$src1,$src2\t# vector (2D)" %}
16342   ins_cost(INSN_COST);
16343   ins_encode %{
16344     __ fmls(as_FloatRegister($dst$$reg), __ T2D,
16345             as_FloatRegister($src1$$reg),
16346             as_FloatRegister($src2$$reg));
16347   %}
16348   ins_pipe(vmuldiv_fp128);
16349 %}
16350 
16351 // --------------------------------- DIV --------------------------------------
16352 
16353 instruct vdiv2F(vecD dst, vecD src1, vecD src2)
16354 %{
16355   predicate(n->as_Vector()->length() == 2);
16356   match(Set dst (DivVF src1 src2));
16357   ins_cost(INSN_COST);
16358   format %{ "fdiv  $dst,$src1,$src2\t# vector (2S)" %}
16359   ins_encode %{
16360     __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
16361             as_FloatRegister($src1$$reg),
16362             as_FloatRegister($src2$$reg));
16363   %}
16364   ins_pipe(vmuldiv_fp64);
16365 %}
16366 
16367 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
16368 %{
16369   predicate(n->as_Vector()->length() == 4);
16370   match(Set dst (DivVF src1 src2));
16371   ins_cost(INSN_COST);
16372   format %{ "fdiv  $dst,$src1,$src2\t# vector (4S)" %}
16373   ins_encode %{
16374     __ fdiv(as_FloatRegister($dst$$reg), __ T4S,
16375             as_FloatRegister($src1$$reg),
16376             as_FloatRegister($src2$$reg));
16377   %}
16378   ins_pipe(vmuldiv_fp128);
16379 %}
16380 
16381 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
16382 %{
16383   predicate(n->as_Vector()->length() == 2);
16384   match(Set dst (DivVD src1 src2));
16385   ins_cost(INSN_COST);
16386   format %{ "fdiv  $dst,$src1,$src2\t# vector (2D)" %}
16387   ins_encode %{
16388     __ fdiv(as_FloatRegister($dst$$reg), __ T2D,
16389             as_FloatRegister($src1$$reg),
16390             as_FloatRegister($src2$$reg));
16391   %}
16392   ins_pipe(vmuldiv_fp128);
16393 %}
16394 
16395 // --------------------------------- SQRT -------------------------------------
16396 
16397 instruct vsqrt2D(vecX dst, vecX src)
16398 %{
16399   predicate(n->as_Vector()->length() == 2);
16400   match(Set dst (SqrtVD src));
16401   format %{ "fsqrt  $dst, $src\t# vector (2D)" %}
16402   ins_encode %{
16403     __ fsqrt(as_FloatRegister($dst$$reg), __ T2D,
16404              as_FloatRegister($src$$reg));
16405   %}
16406   ins_pipe(vsqrt_fp128);
16407 %}
16408 
16409 // --------------------------------- ABS --------------------------------------
16410 
16411 instruct vabs2F(vecD dst, vecD src)
16412 %{
16413   predicate(n->as_Vector()->length() == 2);
16414   match(Set dst (AbsVF src));
16415   ins_cost(INSN_COST * 3);
16416   format %{ "fabs  $dst,$src\t# vector (2S)" %}
16417   ins_encode %{
16418     __ fabs(as_FloatRegister($dst$$reg), __ T2S,
16419             as_FloatRegister($src$$reg));
16420   %}
16421   ins_pipe(vunop_fp64);
16422 %}
16423 
16424 instruct vabs4F(vecX dst, vecX src)
16425 %{
16426   predicate(n->as_Vector()->length() == 4);
16427   match(Set dst (AbsVF src));
16428   ins_cost(INSN_COST * 3);
16429   format %{ "fabs  $dst,$src\t# vector (4S)" %}
16430   ins_encode %{
16431     __ fabs(as_FloatRegister($dst$$reg), __ T4S,
16432             as_FloatRegister($src$$reg));
16433   %}
16434   ins_pipe(vunop_fp128);
16435 %}
16436 
16437 instruct vabs2D(vecX dst, vecX src)
16438 %{
16439   predicate(n->as_Vector()->length() == 2);
16440   match(Set dst (AbsVD src));
16441   ins_cost(INSN_COST * 3);
16442   format %{ "fabs  $dst,$src\t# vector (2D)" %}
16443   ins_encode %{
16444     __ fabs(as_FloatRegister($dst$$reg), __ T2D,
16445             as_FloatRegister($src$$reg));
16446   %}
16447   ins_pipe(vunop_fp128);
16448 %}
16449 
16450 // --------------------------------- NEG --------------------------------------
16451 
16452 instruct vneg2F(vecD dst, vecD src)
16453 %{
16454   predicate(n->as_Vector()->length() == 2);
16455   match(Set dst (NegVF src));
16456   ins_cost(INSN_COST * 3);
16457   format %{ "fneg  $dst,$src\t# vector (2S)" %}
16458   ins_encode %{
16459     __ fneg(as_FloatRegister($dst$$reg), __ T2S,
16460             as_FloatRegister($src$$reg));
16461   %}
16462   ins_pipe(vunop_fp64);
16463 %}
16464 
16465 instruct vneg4F(vecX dst, vecX src)
16466 %{
16467   predicate(n->as_Vector()->length() == 4);
16468   match(Set dst (NegVF src));
16469   ins_cost(INSN_COST * 3);
16470   format %{ "fneg  $dst,$src\t# vector (4S)" %}
16471   ins_encode %{
16472     __ fneg(as_FloatRegister($dst$$reg), __ T4S,
16473             as_FloatRegister($src$$reg));
16474   %}
16475   ins_pipe(vunop_fp128);
16476 %}
16477 
16478 instruct vneg2D(vecX dst, vecX src)
16479 %{
16480   predicate(n->as_Vector()->length() == 2);
16481   match(Set dst (NegVD src));
16482   ins_cost(INSN_COST * 3);
16483   format %{ "fneg  $dst,$src\t# vector (2D)" %}
16484   ins_encode %{
16485     __ fneg(as_FloatRegister($dst$$reg), __ T2D,
16486             as_FloatRegister($src$$reg));
16487   %}
16488   ins_pipe(vunop_fp128);
16489 %}
16490 
16491 // --------------------------------- AND --------------------------------------
16492 
16493 instruct vand8B(vecD dst, vecD src1, vecD src2)
16494 %{
16495   predicate(n->as_Vector()->length_in_bytes() == 4 ||
16496             n->as_Vector()->length_in_bytes() == 8);
16497   match(Set dst (AndV src1 src2));
16498   ins_cost(INSN_COST);
16499   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
16500   ins_encode %{
16501     __ andr(as_FloatRegister($dst$$reg), __ T8B,
16502             as_FloatRegister($src1$$reg),
16503             as_FloatRegister($src2$$reg));
16504   %}
16505   ins_pipe(vlogical64);
16506 %}
16507 
16508 instruct vand16B(vecX dst, vecX src1, vecX src2)
16509 %{
16510   predicate(n->as_Vector()->length_in_bytes() == 16);
16511   match(Set dst (AndV src1 src2));
16512   ins_cost(INSN_COST);
16513   format %{ "and  $dst,$src1,$src2\t# vector (16B)" %}
16514   ins_encode %{
16515     __ andr(as_FloatRegister($dst$$reg), __ T16B,
16516             as_FloatRegister($src1$$reg),
16517             as_FloatRegister($src2$$reg));
16518   %}
16519   ins_pipe(vlogical128);
16520 %}
16521 
16522 // --------------------------------- OR ---------------------------------------
16523 
16524 instruct vor8B(vecD dst, vecD src1, vecD src2)
16525 %{
16526   predicate(n->as_Vector()->length_in_bytes() == 4 ||
16527             n->as_Vector()->length_in_bytes() == 8);
16528   match(Set dst (OrV src1 src2));
16529   ins_cost(INSN_COST);
16530   format %{ "and  $dst,$src1,$src2\t# vector (8B)" %}
16531   ins_encode %{
16532     __ orr(as_FloatRegister($dst$$reg), __ T8B,
16533             as_FloatRegister($src1$$reg),
16534             as_FloatRegister($src2$$reg));
16535   %}
16536   ins_pipe(vlogical64);
16537 %}
16538 
16539 instruct vor16B(vecX dst, vecX src1, vecX src2)
16540 %{
16541   predicate(n->as_Vector()->length_in_bytes() == 16);
16542   match(Set dst (OrV src1 src2));
16543   ins_cost(INSN_COST);
16544   format %{ "orr  $dst,$src1,$src2\t# vector (16B)" %}
16545   ins_encode %{
16546     __ orr(as_FloatRegister($dst$$reg), __ T16B,
16547             as_FloatRegister($src1$$reg),
16548             as_FloatRegister($src2$$reg));
16549   %}
16550   ins_pipe(vlogical128);
16551 %}
16552 
16553 // --------------------------------- XOR --------------------------------------
16554 
16555 instruct vxor8B(vecD dst, vecD src1, vecD src2)
16556 %{
16557   predicate(n->as_Vector()->length_in_bytes() == 4 ||
16558             n->as_Vector()->length_in_bytes() == 8);
16559   match(Set dst (XorV src1 src2));
16560   ins_cost(INSN_COST);
16561   format %{ "xor  $dst,$src1,$src2\t# vector (8B)" %}
16562   ins_encode %{
16563     __ eor(as_FloatRegister($dst$$reg), __ T8B,
16564             as_FloatRegister($src1$$reg),
16565             as_FloatRegister($src2$$reg));
16566   %}
16567   ins_pipe(vlogical64);
16568 %}
16569 
16570 instruct vxor16B(vecX dst, vecX src1, vecX src2)
16571 %{
16572   predicate(n->as_Vector()->length_in_bytes() == 16);
16573   match(Set dst (XorV src1 src2));
16574   ins_cost(INSN_COST);
16575   format %{ "xor  $dst,$src1,$src2\t# vector (16B)" %}
16576   ins_encode %{
16577     __ eor(as_FloatRegister($dst$$reg), __ T16B,
16578             as_FloatRegister($src1$$reg),
16579             as_FloatRegister($src2$$reg));
16580   %}
16581   ins_pipe(vlogical128);
16582 %}
16583 
16584 // ------------------------------ Shift ---------------------------------------
16585 instruct vshiftcnt8B(vecD dst, iRegIorL2I cnt) %{
16586   predicate(n->as_Vector()->length_in_bytes() == 8);
16587   match(Set dst (LShiftCntV cnt));
16588   match(Set dst (RShiftCntV cnt));
16589   format %{ "dup  $dst, $cnt\t# shift count vector (8B)" %}
16590   ins_encode %{
16591     __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($cnt$$reg));
16592   %}
16593   ins_pipe(vdup_reg_reg64);
16594 %}
16595 
16596 instruct vshiftcnt16B(vecX dst, iRegIorL2I cnt) %{
16597   predicate(n->as_Vector()->length_in_bytes() == 16);
16598   match(Set dst (LShiftCntV cnt));
16599   match(Set dst (RShiftCntV cnt));
16600   format %{ "dup  $dst, $cnt\t# shift count vector (16B)" %}
16601   ins_encode %{
16602     __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
16603   %}
16604   ins_pipe(vdup_reg_reg128);
16605 %}
16606 
16607 instruct vsll8B(vecD dst, vecD src, vecD shift) %{
16608   predicate(n->as_Vector()->length() == 4 ||
16609             n->as_Vector()->length() == 8);
16610   match(Set dst (LShiftVB src shift));
16611   ins_cost(INSN_COST);
16612   format %{ "sshl  $dst,$src,$shift\t# vector (8B)" %}
16613   ins_encode %{
16614     __ sshl(as_FloatRegister($dst$$reg), __ T8B,
16615             as_FloatRegister($src$$reg),
16616             as_FloatRegister($shift$$reg));
16617   %}
16618   ins_pipe(vshift64);
16619 %}
16620 
16621 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
16622   predicate(n->as_Vector()->length() == 16);
16623   match(Set dst (LShiftVB src shift));
16624   ins_cost(INSN_COST);
16625   format %{ "sshl  $dst,$src,$shift\t# vector (16B)" %}
16626   ins_encode %{
16627     __ sshl(as_FloatRegister($dst$$reg), __ T16B,
16628             as_FloatRegister($src$$reg),
16629             as_FloatRegister($shift$$reg));
16630   %}
16631   ins_pipe(vshift128);
16632 %}
16633 
16634 // Right shifts with vector shift count on aarch64 SIMD are implemented
16635 // as left shift by negative shift count.
16636 // There are two cases for vector shift count.
16637 //
16638 // Case 1: The vector shift count is from replication.
16639 //        |            |
16640 //    LoadVector  RShiftCntV
16641 //        |       /
16642 //     RShiftVI
16643 // Note: In inner loop, multiple neg instructions are used, which can be
16644 // moved to outer loop and merge into one neg instruction.
16645 //
16646 // Case 2: The vector shift count is from loading.
16647 // This case isn't supported by middle-end now. But it's supported by
16648 // panama/vectorIntrinsics(JEP 338: Vector API).
16649 //        |            |
16650 //    LoadVector  LoadVector
16651 //        |       /
16652 //     RShiftVI
16653 //
16654 
16655 instruct vsra8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
16656   predicate(n->as_Vector()->length() == 4 ||
16657             n->as_Vector()->length() == 8);
16658   match(Set dst (RShiftVB src shift));
16659   ins_cost(INSN_COST);
16660   effect(TEMP tmp);
16661   format %{ "negr  $tmp,$shift\t"
16662             "sshl  $dst,$src,$tmp\t# vector (8B)" %}
16663   ins_encode %{
16664     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16665             as_FloatRegister($shift$$reg));
16666     __ sshl(as_FloatRegister($dst$$reg), __ T8B,
16667             as_FloatRegister($src$$reg),
16668             as_FloatRegister($tmp$$reg));
16669   %}
16670   ins_pipe(vshift64);
16671 %}
16672 
16673 instruct vsra16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
16674   predicate(n->as_Vector()->length() == 16);
16675   match(Set dst (RShiftVB src shift));
16676   ins_cost(INSN_COST);
16677   effect(TEMP tmp);
16678   format %{ "negr  $tmp,$shift\t"
16679             "sshl  $dst,$src,$tmp\t# vector (16B)" %}
16680   ins_encode %{
16681     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16682             as_FloatRegister($shift$$reg));
16683     __ sshl(as_FloatRegister($dst$$reg), __ T16B,
16684             as_FloatRegister($src$$reg),
16685             as_FloatRegister($tmp$$reg));
16686   %}
16687   ins_pipe(vshift128);
16688 %}
16689 
16690 instruct vsrl8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
16691   predicate(n->as_Vector()->length() == 4 ||
16692             n->as_Vector()->length() == 8);
16693   match(Set dst (URShiftVB src shift));
16694   ins_cost(INSN_COST);
16695   effect(TEMP tmp);
16696   format %{ "negr  $tmp,$shift\t"
16697             "ushl  $dst,$src,$tmp\t# vector (8B)" %}
16698   ins_encode %{
16699     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16700             as_FloatRegister($shift$$reg));
16701     __ ushl(as_FloatRegister($dst$$reg), __ T8B,
16702             as_FloatRegister($src$$reg),
16703             as_FloatRegister($tmp$$reg));
16704   %}
16705   ins_pipe(vshift64);
16706 %}
16707 
16708 instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
16709   predicate(n->as_Vector()->length() == 16);
16710   match(Set dst (URShiftVB src shift));
16711   ins_cost(INSN_COST);
16712   effect(TEMP tmp);
16713   format %{ "negr  $tmp,$shift\t"
16714             "ushl  $dst,$src,$tmp\t# vector (16B)" %}
16715   ins_encode %{
16716     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16717             as_FloatRegister($shift$$reg));
16718     __ ushl(as_FloatRegister($dst$$reg), __ T16B,
16719             as_FloatRegister($src$$reg),
16720             as_FloatRegister($tmp$$reg));
16721   %}
16722   ins_pipe(vshift128);
16723 %}
16724 
16725 instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
16726   predicate(n->as_Vector()->length() == 4 ||
16727             n->as_Vector()->length() == 8);
16728   match(Set dst (LShiftVB src shift));
16729   ins_cost(INSN_COST);
16730   format %{ "shl    $dst, $src, $shift\t# vector (8B)" %}
16731   ins_encode %{
16732     int sh = (int)$shift$$constant;
16733     if (sh >= 8) {
16734       __ eor(as_FloatRegister($dst$$reg), __ T8B,
16735              as_FloatRegister($src$$reg),
16736              as_FloatRegister($src$$reg));
16737     } else {
16738       __ shl(as_FloatRegister($dst$$reg), __ T8B,
16739              as_FloatRegister($src$$reg), sh);
16740     }
16741   %}
16742   ins_pipe(vshift64_imm);
16743 %}
16744 
16745 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
16746   predicate(n->as_Vector()->length() == 16);
16747   match(Set dst (LShiftVB src shift));
16748   ins_cost(INSN_COST);
16749   format %{ "shl    $dst, $src, $shift\t# vector (16B)" %}
16750   ins_encode %{
16751     int sh = (int)$shift$$constant;
16752     if (sh >= 8) {
16753       __ eor(as_FloatRegister($dst$$reg), __ T16B,
16754              as_FloatRegister($src$$reg),
16755              as_FloatRegister($src$$reg));
16756     } else {
16757       __ shl(as_FloatRegister($dst$$reg), __ T16B,
16758              as_FloatRegister($src$$reg), sh);
16759     }
16760   %}
16761   ins_pipe(vshift128_imm);
16762 %}
16763 
16764 instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
16765   predicate(n->as_Vector()->length() == 4 ||
16766             n->as_Vector()->length() == 8);
16767   match(Set dst (RShiftVB src shift));
16768   ins_cost(INSN_COST);
16769   format %{ "sshr    $dst, $src, $shift\t# vector (8B)" %}
16770   ins_encode %{
16771     int sh = (int)$shift$$constant;
16772     if (sh >= 8) sh = 7;
16773     __ sshr(as_FloatRegister($dst$$reg), __ T8B,
16774            as_FloatRegister($src$$reg), sh);
16775   %}
16776   ins_pipe(vshift64_imm);
16777 %}
16778 
16779 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
16780   predicate(n->as_Vector()->length() == 16);
16781   match(Set dst (RShiftVB src shift));
16782   ins_cost(INSN_COST);
16783   format %{ "sshr    $dst, $src, $shift\t# vector (16B)" %}
16784   ins_encode %{
16785     int sh = (int)$shift$$constant;
16786     if (sh >= 8) sh = 7;
16787     __ sshr(as_FloatRegister($dst$$reg), __ T16B,
16788            as_FloatRegister($src$$reg), sh);
16789   %}
16790   ins_pipe(vshift128_imm);
16791 %}
16792 
16793 instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
16794   predicate(n->as_Vector()->length() == 4 ||
16795             n->as_Vector()->length() == 8);
16796   match(Set dst (URShiftVB src shift));
16797   ins_cost(INSN_COST);
16798   format %{ "ushr    $dst, $src, $shift\t# vector (8B)" %}
16799   ins_encode %{
16800     int sh = (int)$shift$$constant;
16801     if (sh >= 8) {
16802       __ eor(as_FloatRegister($dst$$reg), __ T8B,
16803              as_FloatRegister($src$$reg),
16804              as_FloatRegister($src$$reg));
16805     } else {
16806       __ ushr(as_FloatRegister($dst$$reg), __ T8B,
16807              as_FloatRegister($src$$reg), sh);
16808     }
16809   %}
16810   ins_pipe(vshift64_imm);
16811 %}
16812 
16813 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
16814   predicate(n->as_Vector()->length() == 16);
16815   match(Set dst (URShiftVB src shift));
16816   ins_cost(INSN_COST);
16817   format %{ "ushr    $dst, $src, $shift\t# vector (16B)" %}
16818   ins_encode %{
16819     int sh = (int)$shift$$constant;
16820     if (sh >= 8) {
16821       __ eor(as_FloatRegister($dst$$reg), __ T16B,
16822              as_FloatRegister($src$$reg),
16823              as_FloatRegister($src$$reg));
16824     } else {
16825       __ ushr(as_FloatRegister($dst$$reg), __ T16B,
16826              as_FloatRegister($src$$reg), sh);
16827     }
16828   %}
16829   ins_pipe(vshift128_imm);
16830 %}
16831 
16832 instruct vsll4S(vecD dst, vecD src, vecD shift) %{
16833   predicate(n->as_Vector()->length() == 2 ||
16834             n->as_Vector()->length() == 4);
16835   match(Set dst (LShiftVS src shift));
16836   ins_cost(INSN_COST);
16837   format %{ "sshl  $dst,$src,$shift\t# vector (4H)" %}
16838   ins_encode %{
16839     __ sshl(as_FloatRegister($dst$$reg), __ T4H,
16840             as_FloatRegister($src$$reg),
16841             as_FloatRegister($shift$$reg));
16842   %}
16843   ins_pipe(vshift64);
16844 %}
16845 
16846 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
16847   predicate(n->as_Vector()->length() == 8);
16848   match(Set dst (LShiftVS src shift));
16849   ins_cost(INSN_COST);
16850   format %{ "sshl  $dst,$src,$shift\t# vector (8H)" %}
16851   ins_encode %{
16852     __ sshl(as_FloatRegister($dst$$reg), __ T8H,
16853             as_FloatRegister($src$$reg),
16854             as_FloatRegister($shift$$reg));
16855   %}
16856   ins_pipe(vshift128);
16857 %}
16858 
16859 instruct vsra4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
16860   predicate(n->as_Vector()->length() == 2 ||
16861             n->as_Vector()->length() == 4);
16862   match(Set dst (RShiftVS src shift));
16863   ins_cost(INSN_COST);
16864   effect(TEMP tmp);
16865   format %{ "negr  $tmp,$shift\t"
16866             "sshl  $dst,$src,$tmp\t# vector (4H)" %}
16867   ins_encode %{
16868     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16869             as_FloatRegister($shift$$reg));
16870     __ sshl(as_FloatRegister($dst$$reg), __ T4H,
16871             as_FloatRegister($src$$reg),
16872             as_FloatRegister($tmp$$reg));
16873   %}
16874   ins_pipe(vshift64);
16875 %}
16876 
16877 instruct vsra8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
16878   predicate(n->as_Vector()->length() == 8);
16879   match(Set dst (RShiftVS src shift));
16880   ins_cost(INSN_COST);
16881   effect(TEMP tmp);
16882   format %{ "negr  $tmp,$shift\t"
16883             "sshl  $dst,$src,$tmp\t# vector (8H)" %}
16884   ins_encode %{
16885     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16886             as_FloatRegister($shift$$reg));
16887     __ sshl(as_FloatRegister($dst$$reg), __ T8H,
16888             as_FloatRegister($src$$reg),
16889             as_FloatRegister($tmp$$reg));
16890   %}
16891   ins_pipe(vshift128);
16892 %}
16893 
16894 instruct vsrl4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
16895   predicate(n->as_Vector()->length() == 2 ||
16896             n->as_Vector()->length() == 4);
16897   match(Set dst (URShiftVS src shift));
16898   ins_cost(INSN_COST);
16899   effect(TEMP tmp);
16900   format %{ "negr  $tmp,$shift\t"
16901             "ushl  $dst,$src,$tmp\t# vector (4H)" %}
16902   ins_encode %{
16903     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
16904             as_FloatRegister($shift$$reg));
16905     __ ushl(as_FloatRegister($dst$$reg), __ T4H,
16906             as_FloatRegister($src$$reg),
16907             as_FloatRegister($tmp$$reg));
16908   %}
16909   ins_pipe(vshift64);
16910 %}
16911 
16912 instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
16913   predicate(n->as_Vector()->length() == 8);
16914   match(Set dst (URShiftVS src shift));
16915   ins_cost(INSN_COST);
16916   effect(TEMP tmp);
16917   format %{ "negr  $tmp,$shift\t"
16918             "ushl  $dst,$src,$tmp\t# vector (8H)" %}
16919   ins_encode %{
16920     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
16921             as_FloatRegister($shift$$reg));
16922     __ ushl(as_FloatRegister($dst$$reg), __ T8H,
16923             as_FloatRegister($src$$reg),
16924             as_FloatRegister($tmp$$reg));
16925   %}
16926   ins_pipe(vshift128);
16927 %}
16928 
16929 instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
16930   predicate(n->as_Vector()->length() == 2 ||
16931             n->as_Vector()->length() == 4);
16932   match(Set dst (LShiftVS src shift));
16933   ins_cost(INSN_COST);
16934   format %{ "shl    $dst, $src, $shift\t# vector (4H)" %}
16935   ins_encode %{
16936     int sh = (int)$shift$$constant;
16937     if (sh >= 16) {
16938       __ eor(as_FloatRegister($dst$$reg), __ T8B,
16939              as_FloatRegister($src$$reg),
16940              as_FloatRegister($src$$reg));
16941     } else {
16942       __ shl(as_FloatRegister($dst$$reg), __ T4H,
16943              as_FloatRegister($src$$reg), sh);
16944     }
16945   %}
16946   ins_pipe(vshift64_imm);
16947 %}
16948 
16949 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
16950   predicate(n->as_Vector()->length() == 8);
16951   match(Set dst (LShiftVS src shift));
16952   ins_cost(INSN_COST);
16953   format %{ "shl    $dst, $src, $shift\t# vector (8H)" %}
16954   ins_encode %{
16955     int sh = (int)$shift$$constant;
16956     if (sh >= 16) {
16957       __ eor(as_FloatRegister($dst$$reg), __ T16B,
16958              as_FloatRegister($src$$reg),
16959              as_FloatRegister($src$$reg));
16960     } else {
16961       __ shl(as_FloatRegister($dst$$reg), __ T8H,
16962              as_FloatRegister($src$$reg), sh);
16963     }
16964   %}
16965   ins_pipe(vshift128_imm);
16966 %}
16967 
16968 instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
16969   predicate(n->as_Vector()->length() == 2 ||
16970             n->as_Vector()->length() == 4);
16971   match(Set dst (RShiftVS src shift));
16972   ins_cost(INSN_COST);
16973   format %{ "sshr    $dst, $src, $shift\t# vector (4H)" %}
16974   ins_encode %{
16975     int sh = (int)$shift$$constant;
16976     if (sh >= 16) sh = 15;
16977     __ sshr(as_FloatRegister($dst$$reg), __ T4H,
16978            as_FloatRegister($src$$reg), sh);
16979   %}
16980   ins_pipe(vshift64_imm);
16981 %}
16982 
16983 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
16984   predicate(n->as_Vector()->length() == 8);
16985   match(Set dst (RShiftVS src shift));
16986   ins_cost(INSN_COST);
16987   format %{ "sshr    $dst, $src, $shift\t# vector (8H)" %}
16988   ins_encode %{
16989     int sh = (int)$shift$$constant;
16990     if (sh >= 16) sh = 15;
16991     __ sshr(as_FloatRegister($dst$$reg), __ T8H,
16992            as_FloatRegister($src$$reg), sh);
16993   %}
16994   ins_pipe(vshift128_imm);
16995 %}
16996 
16997 instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
16998   predicate(n->as_Vector()->length() == 2 ||
16999             n->as_Vector()->length() == 4);
17000   match(Set dst (URShiftVS src shift));
17001   ins_cost(INSN_COST);
17002   format %{ "ushr    $dst, $src, $shift\t# vector (4H)" %}
17003   ins_encode %{
17004     int sh = (int)$shift$$constant;
17005     if (sh >= 16) {
17006       __ eor(as_FloatRegister($dst$$reg), __ T8B,
17007              as_FloatRegister($src$$reg),
17008              as_FloatRegister($src$$reg));
17009     } else {
17010       __ ushr(as_FloatRegister($dst$$reg), __ T4H,
17011              as_FloatRegister($src$$reg), sh);
17012     }
17013   %}
17014   ins_pipe(vshift64_imm);
17015 %}
17016 
17017 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
17018   predicate(n->as_Vector()->length() == 8);
17019   match(Set dst (URShiftVS src shift));
17020   ins_cost(INSN_COST);
17021   format %{ "ushr    $dst, $src, $shift\t# vector (8H)" %}
17022   ins_encode %{
17023     int sh = (int)$shift$$constant;
17024     if (sh >= 16) {
17025       __ eor(as_FloatRegister($dst$$reg), __ T16B,
17026              as_FloatRegister($src$$reg),
17027              as_FloatRegister($src$$reg));
17028     } else {
17029       __ ushr(as_FloatRegister($dst$$reg), __ T8H,
17030              as_FloatRegister($src$$reg), sh);
17031     }
17032   %}
17033   ins_pipe(vshift128_imm);
17034 %}
17035 
17036 instruct vsll2I(vecD dst, vecD src, vecD shift) %{
17037   predicate(n->as_Vector()->length() == 2);
17038   match(Set dst (LShiftVI src shift));
17039   ins_cost(INSN_COST);
17040   format %{ "sshl  $dst,$src,$shift\t# vector (2S)" %}
17041   ins_encode %{
17042     __ sshl(as_FloatRegister($dst$$reg), __ T2S,
17043             as_FloatRegister($src$$reg),
17044             as_FloatRegister($shift$$reg));
17045   %}
17046   ins_pipe(vshift64);
17047 %}
17048 
17049 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
17050   predicate(n->as_Vector()->length() == 4);
17051   match(Set dst (LShiftVI src shift));
17052   ins_cost(INSN_COST);
17053   format %{ "sshl  $dst,$src,$shift\t# vector (4S)" %}
17054   ins_encode %{
17055     __ sshl(as_FloatRegister($dst$$reg), __ T4S,
17056             as_FloatRegister($src$$reg),
17057             as_FloatRegister($shift$$reg));
17058   %}
17059   ins_pipe(vshift128);
17060 %}
17061 
17062 instruct vsra2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
17063   predicate(n->as_Vector()->length() == 2);
17064   match(Set dst (RShiftVI src shift));
17065   ins_cost(INSN_COST);
17066   effect(TEMP tmp);
17067   format %{ "negr  $tmp,$shift\t"
17068             "sshl  $dst,$src,$tmp\t# vector (2S)" %}
17069   ins_encode %{
17070     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17071             as_FloatRegister($shift$$reg));
17072     __ sshl(as_FloatRegister($dst$$reg), __ T2S,
17073             as_FloatRegister($src$$reg),
17074             as_FloatRegister($tmp$$reg));
17075   %}
17076   ins_pipe(vshift64);
17077 %}
17078 
17079 instruct vsra4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
17080   predicate(n->as_Vector()->length() == 4);
17081   match(Set dst (RShiftVI src shift));
17082   ins_cost(INSN_COST);
17083   effect(TEMP tmp);
17084   format %{ "negr  $tmp,$shift\t"
17085             "sshl  $dst,$src,$tmp\t# vector (4S)" %}
17086   ins_encode %{
17087     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17088             as_FloatRegister($shift$$reg));
17089     __ sshl(as_FloatRegister($dst$$reg), __ T4S,
17090             as_FloatRegister($src$$reg),
17091             as_FloatRegister($tmp$$reg));
17092   %}
17093   ins_pipe(vshift128);
17094 %}
17095 
17096 instruct vsrl2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
17097   predicate(n->as_Vector()->length() == 2);
17098   match(Set dst (URShiftVI src shift));
17099   ins_cost(INSN_COST);
17100   effect(TEMP tmp);
17101   format %{ "negr  $tmp,$shift\t"
17102             "ushl  $dst,$src,$tmp\t# vector (2S)" %}
17103   ins_encode %{
17104     __ negr(as_FloatRegister($tmp$$reg), __ T8B,
17105             as_FloatRegister($shift$$reg));
17106     __ ushl(as_FloatRegister($dst$$reg), __ T2S,
17107             as_FloatRegister($src$$reg),
17108             as_FloatRegister($tmp$$reg));
17109   %}
17110   ins_pipe(vshift64);
17111 %}
17112 
17113 instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
17114   predicate(n->as_Vector()->length() == 4);
17115   match(Set dst (URShiftVI src shift));
17116   ins_cost(INSN_COST);
17117   effect(TEMP tmp);
17118   format %{ "negr  $tmp,$shift\t"
17119             "ushl  $dst,$src,$tmp\t# vector (4S)" %}
17120   ins_encode %{
17121     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17122             as_FloatRegister($shift$$reg));
17123     __ ushl(as_FloatRegister($dst$$reg), __ T4S,
17124             as_FloatRegister($src$$reg),
17125             as_FloatRegister($tmp$$reg));
17126   %}
17127   ins_pipe(vshift128);
17128 %}
17129 
17130 instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
17131   predicate(n->as_Vector()->length() == 2);
17132   match(Set dst (LShiftVI src shift));
17133   ins_cost(INSN_COST);
17134   format %{ "shl    $dst, $src, $shift\t# vector (2S)" %}
17135   ins_encode %{
17136     __ shl(as_FloatRegister($dst$$reg), __ T2S,
17137            as_FloatRegister($src$$reg),
17138            (int)$shift$$constant);
17139   %}
17140   ins_pipe(vshift64_imm);
17141 %}
17142 
17143 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
17144   predicate(n->as_Vector()->length() == 4);
17145   match(Set dst (LShiftVI src shift));
17146   ins_cost(INSN_COST);
17147   format %{ "shl    $dst, $src, $shift\t# vector (4S)" %}
17148   ins_encode %{
17149     __ shl(as_FloatRegister($dst$$reg), __ T4S,
17150            as_FloatRegister($src$$reg),
17151            (int)$shift$$constant);
17152   %}
17153   ins_pipe(vshift128_imm);
17154 %}
17155 
17156 instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
17157   predicate(n->as_Vector()->length() == 2);
17158   match(Set dst (RShiftVI src shift));
17159   ins_cost(INSN_COST);
17160   format %{ "sshr    $dst, $src, $shift\t# vector (2S)" %}
17161   ins_encode %{
17162     __ sshr(as_FloatRegister($dst$$reg), __ T2S,
17163             as_FloatRegister($src$$reg),
17164             (int)$shift$$constant);
17165   %}
17166   ins_pipe(vshift64_imm);
17167 %}
17168 
17169 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
17170   predicate(n->as_Vector()->length() == 4);
17171   match(Set dst (RShiftVI src shift));
17172   ins_cost(INSN_COST);
17173   format %{ "sshr    $dst, $src, $shift\t# vector (4S)" %}
17174   ins_encode %{
17175     __ sshr(as_FloatRegister($dst$$reg), __ T4S,
17176             as_FloatRegister($src$$reg),
17177             (int)$shift$$constant);
17178   %}
17179   ins_pipe(vshift128_imm);
17180 %}
17181 
17182 instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
17183   predicate(n->as_Vector()->length() == 2);
17184   match(Set dst (URShiftVI src shift));
17185   ins_cost(INSN_COST);
17186   format %{ "ushr    $dst, $src, $shift\t# vector (2S)" %}
17187   ins_encode %{
17188     __ ushr(as_FloatRegister($dst$$reg), __ T2S,
17189             as_FloatRegister($src$$reg),
17190             (int)$shift$$constant);
17191   %}
17192   ins_pipe(vshift64_imm);
17193 %}
17194 
17195 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
17196   predicate(n->as_Vector()->length() == 4);
17197   match(Set dst (URShiftVI src shift));
17198   ins_cost(INSN_COST);
17199   format %{ "ushr    $dst, $src, $shift\t# vector (4S)" %}
17200   ins_encode %{
17201     __ ushr(as_FloatRegister($dst$$reg), __ T4S,
17202             as_FloatRegister($src$$reg),
17203             (int)$shift$$constant);
17204   %}
17205   ins_pipe(vshift128_imm);
17206 %}
17207 
17208 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
17209   predicate(n->as_Vector()->length() == 2);
17210   match(Set dst (LShiftVL src shift));
17211   ins_cost(INSN_COST);
17212   format %{ "sshl  $dst,$src,$shift\t# vector (2D)" %}
17213   ins_encode %{
17214     __ sshl(as_FloatRegister($dst$$reg), __ T2D,
17215             as_FloatRegister($src$$reg),
17216             as_FloatRegister($shift$$reg));
17217   %}
17218   ins_pipe(vshift128);
17219 %}
17220 
17221 instruct vsra2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
17222   predicate(n->as_Vector()->length() == 2);
17223   match(Set dst (RShiftVL src shift));
17224   ins_cost(INSN_COST);
17225   effect(TEMP tmp);
17226   format %{ "negr  $tmp,$shift\t"
17227             "sshl  $dst,$src,$tmp\t# vector (2D)" %}
17228   ins_encode %{
17229     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17230             as_FloatRegister($shift$$reg));
17231     __ sshl(as_FloatRegister($dst$$reg), __ T2D,
17232             as_FloatRegister($src$$reg),
17233             as_FloatRegister($tmp$$reg));
17234   %}
17235   ins_pipe(vshift128);
17236 %}
17237 
17238 instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
17239   predicate(n->as_Vector()->length() == 2);
17240   match(Set dst (URShiftVL src shift));
17241   ins_cost(INSN_COST);
17242   effect(TEMP tmp);
17243   format %{ "negr  $tmp,$shift\t"
17244             "ushl  $dst,$src,$tmp\t# vector (2D)" %}
17245   ins_encode %{
17246     __ negr(as_FloatRegister($tmp$$reg), __ T16B,
17247             as_FloatRegister($shift$$reg));
17248     __ ushl(as_FloatRegister($dst$$reg), __ T2D,
17249             as_FloatRegister($src$$reg),
17250             as_FloatRegister($tmp$$reg));
17251   %}
17252   ins_pipe(vshift128);
17253 %}
17254 
17255 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
17256   predicate(n->as_Vector()->length() == 2);
17257   match(Set dst (LShiftVL src shift));
17258   ins_cost(INSN_COST);
17259   format %{ "shl    $dst, $src, $shift\t# vector (2D)" %}
17260   ins_encode %{
17261     __ shl(as_FloatRegister($dst$$reg), __ T2D,
17262            as_FloatRegister($src$$reg),
17263            (int)$shift$$constant);
17264   %}
17265   ins_pipe(vshift128_imm);
17266 %}
17267 
17268 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
17269   predicate(n->as_Vector()->length() == 2);
17270   match(Set dst (RShiftVL src shift));
17271   ins_cost(INSN_COST);
17272   format %{ "sshr    $dst, $src, $shift\t# vector (2D)" %}
17273   ins_encode %{
17274     __ sshr(as_FloatRegister($dst$$reg), __ T2D,
17275             as_FloatRegister($src$$reg),
17276             (int)$shift$$constant);
17277   %}
17278   ins_pipe(vshift128_imm);
17279 %}
17280 
17281 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
17282   predicate(n->as_Vector()->length() == 2);
17283   match(Set dst (URShiftVL src shift));
17284   ins_cost(INSN_COST);
17285   format %{ "ushr    $dst, $src, $shift\t# vector (2D)" %}
17286   ins_encode %{
17287     __ ushr(as_FloatRegister($dst$$reg), __ T2D,
17288             as_FloatRegister($src$$reg),
17289             (int)$shift$$constant);
17290   %}
17291   ins_pipe(vshift128_imm);
17292 %}
17293 
17294 instruct vmax2F(vecD dst, vecD src1, vecD src2)
17295 %{
17296   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17297   match(Set dst (MaxV src1 src2));
17298   ins_cost(INSN_COST);
17299   format %{ "fmax  $dst,$src1,$src2\t# vector (2F)" %}
17300   ins_encode %{
17301     __ fmax(as_FloatRegister($dst$$reg), __ T2S,
17302             as_FloatRegister($src1$$reg),
17303             as_FloatRegister($src2$$reg));
17304   %}
17305   ins_pipe(vdop_fp64);
17306 %}
17307 
17308 instruct vmax4F(vecX dst, vecX src1, vecX src2)
17309 %{
17310   predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17311   match(Set dst (MaxV src1 src2));
17312   ins_cost(INSN_COST);
17313   format %{ "fmax  $dst,$src1,$src2\t# vector (4S)" %}
17314   ins_encode %{
17315     __ fmax(as_FloatRegister($dst$$reg), __ T4S,
17316             as_FloatRegister($src1$$reg),
17317             as_FloatRegister($src2$$reg));
17318   %}
17319   ins_pipe(vdop_fp128);
17320 %}
17321 
17322 instruct vmax2D(vecX dst, vecX src1, vecX src2)
17323 %{
17324   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
17325   match(Set dst (MaxV src1 src2));
17326   ins_cost(INSN_COST);
17327   format %{ "fmax  $dst,$src1,$src2\t# vector (2D)" %}
17328   ins_encode %{
17329     __ fmax(as_FloatRegister($dst$$reg), __ T2D,
17330             as_FloatRegister($src1$$reg),
17331             as_FloatRegister($src2$$reg));
17332   %}
17333   ins_pipe(vdop_fp128);
17334 %}
17335 
17336 instruct vmin2F(vecD dst, vecD src1, vecD src2)
17337 %{
17338   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17339   match(Set dst (MinV src1 src2));
17340   ins_cost(INSN_COST);
17341   format %{ "fmin  $dst,$src1,$src2\t# vector (2F)" %}
17342   ins_encode %{
17343     __ fmin(as_FloatRegister($dst$$reg), __ T2S,
17344             as_FloatRegister($src1$$reg),
17345             as_FloatRegister($src2$$reg));
17346   %}
17347   ins_pipe(vdop_fp64);
17348 %}
17349 
17350 instruct vmin4F(vecX dst, vecX src1, vecX src2)
17351 %{
17352   predicate(n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
17353   match(Set dst (MinV src1 src2));
17354   ins_cost(INSN_COST);
17355   format %{ "fmin  $dst,$src1,$src2\t# vector (4S)" %}
17356   ins_encode %{
17357     __ fmin(as_FloatRegister($dst$$reg), __ T4S,
17358             as_FloatRegister($src1$$reg),
17359             as_FloatRegister($src2$$reg));
17360   %}
17361   ins_pipe(vdop_fp128);
17362 %}
17363 
17364 instruct vmin2D(vecX dst, vecX src1, vecX src2)
17365 %{
17366   predicate(n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
17367   match(Set dst (MinV src1 src2));
17368   ins_cost(INSN_COST);
17369   format %{ "fmin  $dst,$src1,$src2\t# vector (2D)" %}
17370   ins_encode %{
17371     __ fmin(as_FloatRegister($dst$$reg), __ T2D,
17372             as_FloatRegister($src1$$reg),
17373             as_FloatRegister($src2$$reg));
17374   %}
17375   ins_pipe(vdop_fp128);
17376 %}
17377 
17378 //----------PEEPHOLE RULES-----------------------------------------------------
17379 // These must follow all instruction definitions as they use the names
17380 // defined in the instructions definitions.
17381 //
17382 // peepmatch ( root_instr_name [preceding_instruction]* );
17383 //
17384 // peepconstraint %{
17385 // (instruction_number.operand_name relational_op instruction_number.operand_name
17386 //  [, ...] );
17387 // // instruction numbers are zero-based using left to right order in peepmatch
17388 //
17389 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17390 // // provide an instruction_number.operand_name for each operand that appears
17391 // // in the replacement instruction's match rule
17392 //
17393 // ---------VM FLAGS---------------------------------------------------------
17394 //
17395 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17396 //
17397 // Each peephole rule is given an identifying number starting with zero and
17398 // increasing by one in the order seen by the parser.  An individual peephole
17399 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17400 // on the command-line.
17401 //
17402 // ---------CURRENT LIMITATIONS----------------------------------------------
17403 //
17404 // Only match adjacent instructions in same basic block
17405 // Only equality constraints
17406 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17407 // Only one replacement instruction
17408 //
17409 // ---------EXAMPLE----------------------------------------------------------
17410 //
17411 // // pertinent parts of existing instructions in architecture description
17412 // instruct movI(iRegINoSp dst, iRegI src)
17413 // %{
17414 //   match(Set dst (CopyI src));
17415 // %}
17416 //
17417 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17418 // %{
17419 //   match(Set dst (AddI dst src));
17420 //   effect(KILL cr);
17421 // %}
17422 //
17423 // // Change (inc mov) to lea
17424 // peephole %{
17425 //   // increment preceeded by register-register move
17426 //   peepmatch ( incI_iReg movI );
17427 //   // require that the destination register of the increment
17428 //   // match the destination register of the move
17429 //   peepconstraint ( 0.dst == 1.dst );
17430 //   // construct a replacement instruction that sets
17431 //   // the destination to ( move's source register + one )
17432 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17433 // %}
17434 //
17435 
17436 // Implementation no longer uses movX instructions since
17437 // machine-independent system no longer uses CopyX nodes.
17438 //
17439 // peephole
17440 // %{
17441 //   peepmatch (incI_iReg movI);
17442 //   peepconstraint (0.dst == 1.dst);
17443 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17444 // %}
17445 
17446 // peephole
17447 // %{
17448 //   peepmatch (decI_iReg movI);
17449 //   peepconstraint (0.dst == 1.dst);
17450 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17451 // %}
17452 
17453 // peephole
17454 // %{
17455 //   peepmatch (addI_iReg_imm movI);
17456 //   peepconstraint (0.dst == 1.dst);
17457 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17458 // %}
17459 
17460 // peephole
17461 // %{
17462 //   peepmatch (incL_iReg movL);
17463 //   peepconstraint (0.dst == 1.dst);
17464 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17465 // %}
17466 
17467 // peephole
17468 // %{
17469 //   peepmatch (decL_iReg movL);
17470 //   peepconstraint (0.dst == 1.dst);
17471 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17472 // %}
17473 
17474 // peephole
17475 // %{
17476 //   peepmatch (addL_iReg_imm movL);
17477 //   peepconstraint (0.dst == 1.dst);
17478 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17479 // %}
17480 
17481 // peephole
17482 // %{
17483 //   peepmatch (addP_iReg_imm movP);
17484 //   peepconstraint (0.dst == 1.dst);
17485 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17486 // %}
17487 
17488 // // Change load of spilled value to only a spill
17489 // instruct storeI(memory mem, iRegI src)
17490 // %{
17491 //   match(Set mem (StoreI mem src));
17492 // %}
17493 //
17494 // instruct loadI(iRegINoSp dst, memory mem)
17495 // %{
17496 //   match(Set dst (LoadI mem));
17497 // %}
17498 //
17499 
17500 //----------SMARTSPILL RULES---------------------------------------------------
17501 // These must follow all instruction definitions as they use the names
17502 // defined in the instructions definitions.
17503 
17504 // Local Variables:
17505 // mode: c++
17506 // End: