1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return 4 * NativeInstruction::instruction_size;
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298   bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // predicate controlling addressing modes
 1315   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1316 %}
 1317 
 1318 source %{
 1319 
 1320   // Derived RegMask with conditionally allocatable registers
 1321 
 1322   void PhaseOutput::pd_perform_mach_node_analysis() {
 1323   }
 1324 
 1325   int MachNode::pd_alignment_required() const {
 1326     return 1;
 1327   }
 1328 
 1329   int MachNode::compute_padding(int current_offset) const {
 1330     return 0;
 1331   }
 1332 
 1333   RegMask _ANY_REG32_mask;
 1334   RegMask _ANY_REG_mask;
 1335   RegMask _PTR_REG_mask;
 1336   RegMask _NO_SPECIAL_REG32_mask;
 1337   RegMask _NO_SPECIAL_REG_mask;
 1338   RegMask _NO_SPECIAL_PTR_REG_mask;
 1339 
 1340   void reg_mask_init() {
 1341     // We derive below RegMask(s) from the ones which are auto-generated from
 1342     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1343     // registers conditionally reserved.
 1344 
 1345     _ANY_REG32_mask = _ALL_REG32_mask;
 1346     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1347 
 1348     _ANY_REG_mask = _ALL_REG_mask;
 1349 
 1350     _PTR_REG_mask = _ALL_REG_mask;
 1351 
 1352     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1353     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1354 
 1355     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1356     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1357 
 1358     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     // r27 is not allocatable when compressed oops is on and heapbase is not
 1362     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1363     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1364       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1365       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1366       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1367     }
 1368 
 1369     // r29 is not allocatable when PreserveFramePointer is on
 1370     if (PreserveFramePointer) {
 1371       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1372       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1373       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1374     }
 1375   }
 1376 
 1377   // Optimizaton of volatile gets and puts
 1378   // -------------------------------------
 1379   //
 1380   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1381   // use to implement volatile reads and writes. For a volatile read
 1382   // we simply need
 1383   //
 1384   //   ldar<x>
 1385   //
 1386   // and for a volatile write we need
 1387   //
 1388   //   stlr<x>
 1389   //
 1390   // Alternatively, we can implement them by pairing a normal
 1391   // load/store with a memory barrier. For a volatile read we need
 1392   //
 1393   //   ldr<x>
 1394   //   dmb ishld
 1395   //
 1396   // for a volatile write
 1397   //
 1398   //   dmb ish
 1399   //   str<x>
 1400   //   dmb ish
 1401   //
 1402   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1403   // sequences. These are normally translated to an instruction
 1404   // sequence like the following
 1405   //
 1406   //   dmb      ish
 1407   // retry:
 1408   //   ldxr<x>   rval raddr
 1409   //   cmp       rval rold
 1410   //   b.ne done
 1411   //   stlxr<x>  rval, rnew, rold
 1412   //   cbnz      rval retry
 1413   // done:
 1414   //   cset      r0, eq
 1415   //   dmb ishld
 1416   //
 1417   // Note that the exclusive store is already using an stlxr
 1418   // instruction. That is required to ensure visibility to other
 1419   // threads of the exclusive write (assuming it succeeds) before that
 1420   // of any subsequent writes.
 1421   //
 1422   // The following instruction sequence is an improvement on the above
 1423   //
 1424   // retry:
 1425   //   ldaxr<x>  rval raddr
 1426   //   cmp       rval rold
 1427   //   b.ne done
 1428   //   stlxr<x>  rval, rnew, rold
 1429   //   cbnz      rval retry
 1430   // done:
 1431   //   cset      r0, eq
 1432   //
 1433   // We don't need the leading dmb ish since the stlxr guarantees
 1434   // visibility of prior writes in the case that the swap is
 1435   // successful. Crucially we don't have to worry about the case where
 1436   // the swap is not successful since no valid program should be
 1437   // relying on visibility of prior changes by the attempting thread
 1438   // in the case where the CAS fails.
 1439   //
 1440   // Similarly, we don't need the trailing dmb ishld if we substitute
 1441   // an ldaxr instruction since that will provide all the guarantees we
 1442   // require regarding observation of changes made by other threads
 1443   // before any change to the CAS address observed by the load.
 1444   //
 1445   // In order to generate the desired instruction sequence we need to
 1446   // be able to identify specific 'signature' ideal graph node
 1447   // sequences which i) occur as a translation of a volatile reads or
 1448   // writes or CAS operations and ii) do not occur through any other
 1449   // translation or graph transformation. We can then provide
 1450   // alternative aldc matching rules which translate these node
 1451   // sequences to the desired machine code sequences. Selection of the
 1452   // alternative rules can be implemented by predicates which identify
 1453   // the relevant node sequences.
 1454   //
 1455   // The ideal graph generator translates a volatile read to the node
 1456   // sequence
 1457   //
 1458   //   LoadX[mo_acquire]
 1459   //   MemBarAcquire
 1460   //
 1461   // As a special case when using the compressed oops optimization we
 1462   // may also see this variant
 1463   //
 1464   //   LoadN[mo_acquire]
 1465   //   DecodeN
 1466   //   MemBarAcquire
 1467   //
 1468   // A volatile write is translated to the node sequence
 1469   //
 1470   //   MemBarRelease
 1471   //   StoreX[mo_release] {CardMark}-optional
 1472   //   MemBarVolatile
 1473   //
 1474   // n.b. the above node patterns are generated with a strict
 1475   // 'signature' configuration of input and output dependencies (see
 1476   // the predicates below for exact details). The card mark may be as
 1477   // simple as a few extra nodes or, in a few GC configurations, may
 1478   // include more complex control flow between the leading and
 1479   // trailing memory barriers. However, whatever the card mark
 1480   // configuration these signatures are unique to translated volatile
 1481   // reads/stores -- they will not appear as a result of any other
 1482   // bytecode translation or inlining nor as a consequence of
 1483   // optimizing transforms.
 1484   //
 1485   // We also want to catch inlined unsafe volatile gets and puts and
 1486   // be able to implement them using either ldar<x>/stlr<x> or some
 1487   // combination of ldr<x>/stlr<x> and dmb instructions.
 1488   //
 1489   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1490   // normal volatile put node sequence containing an extra cpuorder
 1491   // membar
 1492   //
 1493   //   MemBarRelease
 1494   //   MemBarCPUOrder
 1495   //   StoreX[mo_release] {CardMark}-optional
 1496   //   MemBarCPUOrder
 1497   //   MemBarVolatile
 1498   //
 1499   // n.b. as an aside, a cpuorder membar is not itself subject to
 1500   // matching and translation by adlc rules.  However, the rule
 1501   // predicates need to detect its presence in order to correctly
 1502   // select the desired adlc rules.
 1503   //
 1504   // Inlined unsafe volatile gets manifest as a slightly different
 1505   // node sequence to a normal volatile get because of the
 1506   // introduction of some CPUOrder memory barriers to bracket the
 1507   // Load. However, but the same basic skeleton of a LoadX feeding a
 1508   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1509   // present
 1510   //
 1511   //   MemBarCPUOrder
 1512   //        ||       \\
 1513   //   MemBarCPUOrder LoadX[mo_acquire]
 1514   //        ||            |
 1515   //        ||       {DecodeN} optional
 1516   //        ||       /
 1517   //     MemBarAcquire
 1518   //
 1519   // In this case the acquire membar does not directly depend on the
 1520   // load. However, we can be sure that the load is generated from an
 1521   // inlined unsafe volatile get if we see it dependent on this unique
 1522   // sequence of membar nodes. Similarly, given an acquire membar we
 1523   // can know that it was added because of an inlined unsafe volatile
 1524   // get if it is fed and feeds a cpuorder membar and if its feed
 1525   // membar also feeds an acquiring load.
 1526   //
 1527   // Finally an inlined (Unsafe) CAS operation is translated to the
 1528   // following ideal graph
 1529   //
 1530   //   MemBarRelease
 1531   //   MemBarCPUOrder
 1532   //   CompareAndSwapX {CardMark}-optional
 1533   //   MemBarCPUOrder
 1534   //   MemBarAcquire
 1535   //
 1536   // So, where we can identify these volatile read and write
 1537   // signatures we can choose to plant either of the above two code
 1538   // sequences. For a volatile read we can simply plant a normal
 1539   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1540   // also choose to inhibit translation of the MemBarAcquire and
 1541   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1542   //
 1543   // When we recognise a volatile store signature we can choose to
 1544   // plant at a dmb ish as a translation for the MemBarRelease, a
 1545   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1546   // Alternatively, we can inhibit translation of the MemBarRelease
 1547   // and MemBarVolatile and instead plant a simple stlr<x>
 1548   // instruction.
 1549   //
 1550   // when we recognise a CAS signature we can choose to plant a dmb
 1551   // ish as a translation for the MemBarRelease, the conventional
 1552   // macro-instruction sequence for the CompareAndSwap node (which
 1553   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1554   // Alternatively, we can elide generation of the dmb instructions
 1555   // and plant the alternative CompareAndSwap macro-instruction
 1556   // sequence (which uses ldaxr<x>).
 1557   //
 1558   // Of course, the above only applies when we see these signature
 1559   // configurations. We still want to plant dmb instructions in any
 1560   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1561   // MemBarVolatile. For example, at the end of a constructor which
 1562   // writes final/volatile fields we will see a MemBarRelease
 1563   // instruction and this needs a 'dmb ish' lest we risk the
 1564   // constructed object being visible without making the
 1565   // final/volatile field writes visible.
 1566   //
 1567   // n.b. the translation rules below which rely on detection of the
 1568   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1569   // If we see anything other than the signature configurations we
 1570   // always just translate the loads and stores to ldr<x> and str<x>
 1571   // and translate acquire, release and volatile membars to the
 1572   // relevant dmb instructions.
 1573   //
 1574 
 1575   // is_CAS(int opcode, bool maybe_volatile)
 1576   //
 1577   // return true if opcode is one of the possible CompareAndSwapX
 1578   // values otherwise false.
 1579 
 1580   bool is_CAS(int opcode, bool maybe_volatile)
 1581   {
 1582     switch(opcode) {
 1583       // We handle these
 1584     case Op_CompareAndSwapI:
 1585     case Op_CompareAndSwapL:
 1586     case Op_CompareAndSwapP:
 1587     case Op_CompareAndSwapN:
 1588     case Op_ShenandoahCompareAndSwapP:
 1589     case Op_ShenandoahCompareAndSwapN:
 1590     case Op_CompareAndSwapB:
 1591     case Op_CompareAndSwapS:
 1592     case Op_GetAndSetI:
 1593     case Op_GetAndSetL:
 1594     case Op_GetAndSetP:
 1595     case Op_GetAndSetN:
 1596     case Op_GetAndAddI:
 1597     case Op_GetAndAddL:
 1598       return true;
 1599     case Op_CompareAndExchangeI:
 1600     case Op_CompareAndExchangeN:
 1601     case Op_CompareAndExchangeB:
 1602     case Op_CompareAndExchangeS:
 1603     case Op_CompareAndExchangeL:
 1604     case Op_CompareAndExchangeP:
 1605     case Op_WeakCompareAndSwapB:
 1606     case Op_WeakCompareAndSwapS:
 1607     case Op_WeakCompareAndSwapI:
 1608     case Op_WeakCompareAndSwapL:
 1609     case Op_WeakCompareAndSwapP:
 1610     case Op_WeakCompareAndSwapN:
 1611     case Op_ShenandoahWeakCompareAndSwapP:
 1612     case Op_ShenandoahWeakCompareAndSwapN:
 1613     case Op_ShenandoahCompareAndExchangeP:
 1614     case Op_ShenandoahCompareAndExchangeN:
 1615       return maybe_volatile;
 1616     default:
 1617       return false;
 1618     }
 1619   }
 1620 
 1621   // helper to determine the maximum number of Phi nodes we may need to
 1622   // traverse when searching from a card mark membar for the merge mem
 1623   // feeding a trailing membar or vice versa
 1624 
 1625 // predicates controlling emit of ldr<x>/ldar<x>
 1626 
 1627 bool unnecessary_acquire(const Node *barrier)
 1628 {
 1629   assert(barrier->is_MemBar(), "expecting a membar");
 1630 
 1631   MemBarNode* mb = barrier->as_MemBar();
 1632 
 1633   if (mb->trailing_load()) {
 1634     return true;
 1635   }
 1636 
 1637   if (mb->trailing_load_store()) {
 1638     Node* load_store = mb->in(MemBarNode::Precedent);
 1639     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1640     return is_CAS(load_store->Opcode(), true);
 1641   }
 1642 
 1643   return false;
 1644 }
 1645 
 1646 bool needs_acquiring_load(const Node *n)
 1647 {
 1648   assert(n->is_Load(), "expecting a load");
 1649   LoadNode *ld = n->as_Load();
 1650   return ld->is_acquire();
 1651 }
 1652 
 1653 bool unnecessary_release(const Node *n)
 1654 {
 1655   assert((n->is_MemBar() &&
 1656           n->Opcode() == Op_MemBarRelease),
 1657          "expecting a release membar");
 1658 
 1659   MemBarNode *barrier = n->as_MemBar();
 1660   if (!barrier->leading()) {
 1661     return false;
 1662   } else {
 1663     Node* trailing = barrier->trailing_membar();
 1664     MemBarNode* trailing_mb = trailing->as_MemBar();
 1665     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1666     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1667 
 1668     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1669     if (mem->is_Store()) {
 1670       assert(mem->as_Store()->is_release(), "");
 1671       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1672       return true;
 1673     } else {
 1674       assert(mem->is_LoadStore(), "");
 1675       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1676       return is_CAS(mem->Opcode(), true);
 1677     }
 1678   }
 1679   return false;
 1680 }
 1681 
 1682 bool unnecessary_volatile(const Node *n)
 1683 {
 1684   // assert n->is_MemBar();
 1685   MemBarNode *mbvol = n->as_MemBar();
 1686 
 1687   bool release = mbvol->trailing_store();
 1688   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1689 #ifdef ASSERT
 1690   if (release) {
 1691     Node* leading = mbvol->leading_membar();
 1692     assert(leading->Opcode() == Op_MemBarRelease, "");
 1693     assert(leading->as_MemBar()->leading_store(), "");
 1694     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1695   }
 1696 #endif
 1697 
 1698   return release;
 1699 }
 1700 
 1701 // predicates controlling emit of str<x>/stlr<x>
 1702 
 1703 bool needs_releasing_store(const Node *n)
 1704 {
 1705   // assert n->is_Store();
 1706   StoreNode *st = n->as_Store();
 1707   return st->trailing_membar() != NULL;
 1708 }
 1709 
 1710 // predicate controlling translation of CAS
 1711 //
 1712 // returns true if CAS needs to use an acquiring load otherwise false
 1713 
 1714 bool needs_acquiring_load_exclusive(const Node *n)
 1715 {
 1716   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1717   LoadStoreNode* ldst = n->as_LoadStore();
 1718   if (is_CAS(n->Opcode(), false)) {
 1719     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1720   } else {
 1721     return ldst->trailing_membar() != NULL;
 1722   }
 1723 
 1724   // so we can just return true here
 1725   return true;
 1726 }
 1727 
 1728 #define __ _masm.
 1729 
 1730 // advance declarations for helper functions to convert register
 1731 // indices to register objects
 1732 
 1733 // the ad file has to provide implementations of certain methods
 1734 // expected by the generic code
 1735 //
 1736 // REQUIRED FUNCTIONALITY
 1737 
 1738 //=============================================================================
 1739 
 1740 // !!!!! Special hack to get all types of calls to specify the byte offset
 1741 //       from the start of the call to the point where the return address
 1742 //       will point.
 1743 
 1744 int MachCallStaticJavaNode::ret_addr_offset()
 1745 {
 1746   // call should be a simple bl
 1747   int off = 4;
 1748   return off;
 1749 }
 1750 
 1751 int MachCallDynamicJavaNode::ret_addr_offset()
 1752 {
 1753   return 16; // movz, movk, movk, bl
 1754 }
 1755 
 1756 int MachCallRuntimeNode::ret_addr_offset() {
 1757   // for generated stubs the call will be
 1758   //   bl(addr)
 1759   // or with far branches
 1760   //   bl(trampoline_stub)
 1761   // for real runtime callouts it will be six instructions
 1762   // see aarch64_enc_java_to_runtime
 1763   //   adr(rscratch2, retaddr)
 1764   //   lea(rscratch1, RuntimeAddress(addr)
 1765   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1766   //   blr(rscratch1)
 1767   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1768   if (cb) {
 1769     return 1 * NativeInstruction::instruction_size;
 1770   } else {
 1771     return 6 * NativeInstruction::instruction_size;
 1772   }
 1773 }
 1774 
 1775 int MachCallNativeNode::ret_addr_offset() {
 1776   // This is implemented using aarch64_enc_java_to_runtime as above.
 1777   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1778   if (cb) {
 1779     return 1 * NativeInstruction::instruction_size;
 1780   } else {
 1781     return 6 * NativeInstruction::instruction_size;
 1782   }
 1783 }
 1784 
 1785 //=============================================================================
 1786 
 1787 #ifndef PRODUCT
 1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1789   st->print("BREAKPOINT");
 1790 }
 1791 #endif
 1792 
 1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1794   C2_MacroAssembler _masm(&cbuf);
 1795   __ brk(0);
 1796 }
 1797 
 1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1799   return MachNode::size(ra_);
 1800 }
 1801 
 1802 //=============================================================================
 1803 
 1804 #ifndef PRODUCT
 1805   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1806     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1807   }
 1808 #endif
 1809 
 1810   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1811     C2_MacroAssembler _masm(&cbuf);
 1812     for (int i = 0; i < _count; i++) {
 1813       __ nop();
 1814     }
 1815   }
 1816 
 1817   uint MachNopNode::size(PhaseRegAlloc*) const {
 1818     return _count * NativeInstruction::instruction_size;
 1819   }
 1820 
 1821 //=============================================================================
 1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1823 
 1824 int ConstantTable::calculate_table_base_offset() const {
 1825   return 0;  // absolute addressing, no offset
 1826 }
 1827 
 1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1830   ShouldNotReachHere();
 1831 }
 1832 
 1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1834   // Empty encoding
 1835 }
 1836 
 1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1838   return 0;
 1839 }
 1840 
 1841 #ifndef PRODUCT
 1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1843   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1844 }
 1845 #endif
 1846 
 1847 #ifndef PRODUCT
 1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1849   Compile* C = ra_->C;
 1850 
 1851   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1852 
 1853   if (C->output()->need_stack_bang(framesize))
 1854     st->print("# stack bang size=%d\n\t", framesize);
 1855 
 1856   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1857     st->print("sub  sp, sp, #%d\n\t", framesize);
 1858     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1859     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1860   } else {
 1861     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1862     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1863     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1864     st->print("sub  sp, sp, rscratch1");
 1865   }
 1866   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1867     st->print("\n\t");
 1868     st->print("ldr  rscratch1, [guard]\n\t");
 1869     st->print("dmb ishld\n\t");
 1870     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1871     st->print("cmp  rscratch1, rscratch2\n\t");
 1872     st->print("b.eq skip");
 1873     st->print("\n\t");
 1874     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1875     st->print("b skip\n\t");
 1876     st->print("guard: int\n\t");
 1877     st->print("\n\t");
 1878     st->print("skip:\n\t");
 1879   }
 1880 }
 1881 #endif
 1882 
 1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1884   Compile* C = ra_->C;
 1885   C2_MacroAssembler _masm(&cbuf);
 1886 
 1887   // n.b. frame size includes space for return pc and rfp
 1888   const int framesize = C->output()->frame_size_in_bytes();
 1889 
 1890   // insert a nop at the start of the prolog so we can patch in a
 1891   // branch if we need to invalidate the method later
 1892   __ nop();
 1893 
 1894   if (C->clinit_barrier_on_entry()) {
 1895     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1896 
 1897     Label L_skip_barrier;
 1898 
 1899     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1900     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1901     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1902     __ bind(L_skip_barrier);
 1903   }
 1904 
 1905   if (C->max_vector_size() > 0) {
 1906     __ reinitialize_ptrue();
 1907   }
 1908 
 1909   int bangsize = C->output()->bang_size_in_bytes();
 1910   if (C->output()->need_stack_bang(bangsize))
 1911     __ generate_stack_overflow_check(bangsize);
 1912 
 1913   __ build_frame(framesize);
 1914 
 1915   if (C->stub_function() == NULL) {
 1916     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1917     bs->nmethod_entry_barrier(&_masm);
 1918   }
 1919 
 1920   if (VerifyStackAtCalls) {
 1921     Unimplemented();
 1922   }
 1923 
 1924   C->output()->set_frame_complete(cbuf.insts_size());
 1925 
 1926   if (C->has_mach_constant_base_node()) {
 1927     // NOTE: We set the table base offset here because users might be
 1928     // emitted before MachConstantBaseNode.
 1929     ConstantTable& constant_table = C->output()->constant_table();
 1930     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1931   }
 1932 }
 1933 
 1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1935 {
 1936   return MachNode::size(ra_); // too many variables; just compute it
 1937                               // the hard way
 1938 }
 1939 
 1940 int MachPrologNode::reloc() const
 1941 {
 1942   return 0;
 1943 }
 1944 
 1945 //=============================================================================
 1946 
 1947 #ifndef PRODUCT
 1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1949   Compile* C = ra_->C;
 1950   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1951 
 1952   st->print("# pop frame %d\n\t",framesize);
 1953 
 1954   if (framesize == 0) {
 1955     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1956   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1957     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1958     st->print("add  sp, sp, #%d\n\t", framesize);
 1959   } else {
 1960     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, rscratch1\n\t");
 1962     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1963   }
 1964 
 1965   if (do_polling() && C->is_method_compilation()) {
 1966     st->print("# test polling word\n\t");
 1967     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1968     st->print("cmp  sp, rscratch1\n\t");
 1969     st->print("bhi #slow_path");
 1970   }
 1971 }
 1972 #endif
 1973 
 1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1975   Compile* C = ra_->C;
 1976   C2_MacroAssembler _masm(&cbuf);
 1977   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1978 
 1979   __ remove_frame(framesize);
 1980 
 1981   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1982     __ reserved_stack_check();
 1983   }
 1984 
 1985   if (do_polling() && C->is_method_compilation()) {
 1986     Label dummy_label;
 1987     Label* code_stub = &dummy_label;
 1988     if (!C->output()->in_scratch_emit_size()) {
 1989       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1990     }
 1991     __ relocate(relocInfo::poll_return_type);
 1992     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1993   }
 1994 }
 1995 
 1996 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1997   // Variable size. Determine dynamically.
 1998   return MachNode::size(ra_);
 1999 }
 2000 
 2001 int MachEpilogNode::reloc() const {
 2002   // Return number of relocatable values contained in this instruction.
 2003   return 1; // 1 for polling page.
 2004 }
 2005 
 2006 const Pipeline * MachEpilogNode::pipeline() const {
 2007   return MachNode::pipeline_class();
 2008 }
 2009 
 2010 //=============================================================================
 2011 
 2012 // Figure out which register class each belongs in: rc_int, rc_float or
 2013 // rc_stack.
 2014 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2015 
 2016 static enum RC rc_class(OptoReg::Name reg) {
 2017 
 2018   if (reg == OptoReg::Bad) {
 2019     return rc_bad;
 2020   }
 2021 
 2022   // we have 32 int registers * 2 halves
 2023   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2024 
 2025   if (reg < slots_of_int_registers) {
 2026     return rc_int;
 2027   }
 2028 
 2029   // we have 32 float register * 8 halves
 2030   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2031   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2032     return rc_float;
 2033   }
 2034 
 2035   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2036   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2037     return rc_predicate;
 2038   }
 2039 
 2040   // Between predicate regs & stack is the flags.
 2041   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2042 
 2043   return rc_stack;
 2044 }
 2045 
 2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2047   Compile* C = ra_->C;
 2048 
 2049   // Get registers to move.
 2050   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2051   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2052   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2053   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2054 
 2055   enum RC src_hi_rc = rc_class(src_hi);
 2056   enum RC src_lo_rc = rc_class(src_lo);
 2057   enum RC dst_hi_rc = rc_class(dst_hi);
 2058   enum RC dst_lo_rc = rc_class(dst_lo);
 2059 
 2060   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2061 
 2062   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 2063     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2064            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2065            "expected aligned-adjacent pairs");
 2066   }
 2067 
 2068   if (src_lo == dst_lo && src_hi == dst_hi) {
 2069     return 0;            // Self copy, no move.
 2070   }
 2071 
 2072   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2073               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2074   int src_offset = ra_->reg2offset(src_lo);
 2075   int dst_offset = ra_->reg2offset(dst_lo);
 2076 
 2077   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2078     uint ireg = ideal_reg();
 2079     if (ireg == Op_VecA && cbuf) {
 2080       C2_MacroAssembler _masm(cbuf);
 2081       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2082       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2083         // stack->stack
 2084         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2085                                                 sve_vector_reg_size_in_bytes);
 2086       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2087         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2088                             sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2090         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2091                               sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2093         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2094                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2095                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2096       } else {
 2097         ShouldNotReachHere();
 2098       }
 2099     } else if (cbuf) {
 2100       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2101       C2_MacroAssembler _masm(cbuf);
 2102       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2103       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2104         // stack->stack
 2105         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2106         if (ireg == Op_VecD) {
 2107           __ unspill(rscratch1, true, src_offset);
 2108           __ spill(rscratch1, true, dst_offset);
 2109         } else {
 2110           __ spill_copy128(src_offset, dst_offset);
 2111         }
 2112       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2113         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2114                ireg == Op_VecD ? __ T8B : __ T16B,
 2115                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2116       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2117         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2118                  ireg == Op_VecD ? __ D : __ Q,
 2119                  ra_->reg2offset(dst_lo));
 2120       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2121         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2122                    ireg == Op_VecD ? __ D : __ Q,
 2123                    ra_->reg2offset(src_lo));
 2124       } else {
 2125         ShouldNotReachHere();
 2126       }
 2127     }
 2128   } else if (cbuf) {
 2129     C2_MacroAssembler _masm(cbuf);
 2130     switch (src_lo_rc) {
 2131     case rc_int:
 2132       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2133         if (is64) {
 2134             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2135                    as_Register(Matcher::_regEncode[src_lo]));
 2136         } else {
 2137             C2_MacroAssembler _masm(cbuf);
 2138             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2139                     as_Register(Matcher::_regEncode[src_lo]));
 2140         }
 2141       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2142         if (is64) {
 2143             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2144                      as_Register(Matcher::_regEncode[src_lo]));
 2145         } else {
 2146             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2147                      as_Register(Matcher::_regEncode[src_lo]));
 2148         }
 2149       } else {                    // gpr --> stack spill
 2150         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2151         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2152       }
 2153       break;
 2154     case rc_float:
 2155       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2156         if (is64) {
 2157             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2158                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2159         } else {
 2160             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2161                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2162         }
 2163       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2164         if (is64) {
 2165             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2166                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2167         } else {
 2168             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         }
 2171       } else {                    // fpr --> stack spill
 2172         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2173         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2174                  is64 ? __ D : __ S, dst_offset);
 2175       }
 2176       break;
 2177     case rc_stack:
 2178       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2179         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2180       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2181         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2182                    is64 ? __ D : __ S, src_offset);
 2183       } else if (dst_lo_rc == rc_predicate) {
 2184         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2185                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2186       } else {                    // stack --> stack copy
 2187         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2188         if (ideal_reg() == Op_RegVectMask) {
 2189           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2190                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2191         } else {
 2192           __ unspill(rscratch1, is64, src_offset);
 2193           __ spill(rscratch1, is64, dst_offset);
 2194         }
 2195       }
 2196       break;
 2197     case rc_predicate:
 2198       if (dst_lo_rc == rc_predicate) {
 2199         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2200       } else if (dst_lo_rc == rc_stack) {
 2201         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2202                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2203       } else {
 2204         assert(false, "bad src and dst rc_class combination.");
 2205         ShouldNotReachHere();
 2206       }
 2207       break;
 2208     default:
 2209       assert(false, "bad rc_class for spill");
 2210       ShouldNotReachHere();
 2211     }
 2212   }
 2213 
 2214   if (st) {
 2215     st->print("spill ");
 2216     if (src_lo_rc == rc_stack) {
 2217       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2218     } else {
 2219       st->print("%s -> ", Matcher::regName[src_lo]);
 2220     }
 2221     if (dst_lo_rc == rc_stack) {
 2222       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2223     } else {
 2224       st->print("%s", Matcher::regName[dst_lo]);
 2225     }
 2226     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2227       int vsize = 0;
 2228       switch (ideal_reg()) {
 2229       case Op_VecD:
 2230         vsize = 64;
 2231         break;
 2232       case Op_VecX:
 2233         vsize = 128;
 2234         break;
 2235       case Op_VecA:
 2236         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2237         break;
 2238       default:
 2239         assert(false, "bad register type for spill");
 2240         ShouldNotReachHere();
 2241       }
 2242       st->print("\t# vector spill size = %d", vsize);
 2243     } else if (ideal_reg() == Op_RegVectMask) {
 2244       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2245       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2246       st->print("\t# predicate spill size = %d", vsize);
 2247     } else {
 2248       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2249     }
 2250   }
 2251 
 2252   return 0;
 2253 
 2254 }
 2255 
 2256 #ifndef PRODUCT
 2257 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2258   if (!ra_)
 2259     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2260   else
 2261     implementation(NULL, ra_, false, st);
 2262 }
 2263 #endif
 2264 
 2265 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2266   implementation(&cbuf, ra_, false, NULL);
 2267 }
 2268 
 2269 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2270   return MachNode::size(ra_);
 2271 }
 2272 
 2273 //=============================================================================
 2274 
 2275 #ifndef PRODUCT
 2276 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2277   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2278   int reg = ra_->get_reg_first(this);
 2279   st->print("add %s, rsp, #%d]\t# box lock",
 2280             Matcher::regName[reg], offset);
 2281 }
 2282 #endif
 2283 
 2284 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2285   C2_MacroAssembler _masm(&cbuf);
 2286 
 2287   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2288   int reg    = ra_->get_encode(this);
 2289 
 2290   // This add will handle any 24-bit signed offset. 24 bits allows an
 2291   // 8 megabyte stack frame.
 2292   __ add(as_Register(reg), sp, offset);
 2293 }
 2294 
 2295 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2296   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2297   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2298 
 2299   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2300     return NativeInstruction::instruction_size;
 2301   } else {
 2302     return 2 * NativeInstruction::instruction_size;
 2303   }
 2304 }
 2305 
 2306 //=============================================================================
 2307 
 2308 #ifndef PRODUCT
 2309 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2310 {
 2311   st->print_cr("# MachUEPNode");
 2312   if (UseCompressedClassPointers) {
 2313     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2314     if (CompressedKlassPointers::shift() != 0) {
 2315       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2316     }
 2317   } else {
 2318    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2319   }
 2320   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2321   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2322 }
 2323 #endif
 2324 
 2325 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2326 {
 2327   // This is the unverified entry point.
 2328   C2_MacroAssembler _masm(&cbuf);
 2329 
 2330   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2331   Label skip;
 2332   // TODO
 2333   // can we avoid this skip and still use a reloc?
 2334   __ br(Assembler::EQ, skip);
 2335   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2336   __ bind(skip);
 2337 }
 2338 
 2339 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2340 {
 2341   return MachNode::size(ra_);
 2342 }
 2343 
 2344 // REQUIRED EMIT CODE
 2345 
 2346 //=============================================================================
 2347 
 2348 // Emit exception handler code.
 2349 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2350 {
 2351   // mov rscratch1 #exception_blob_entry_point
 2352   // br rscratch1
 2353   // Note that the code buffer's insts_mark is always relative to insts.
 2354   // That's why we must use the macroassembler to generate a handler.
 2355   C2_MacroAssembler _masm(&cbuf);
 2356   address base = __ start_a_stub(size_exception_handler());
 2357   if (base == NULL) {
 2358     ciEnv::current()->record_failure("CodeCache is full");
 2359     return 0;  // CodeBuffer::expand failed
 2360   }
 2361   int offset = __ offset();
 2362   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2363   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2364   __ end_a_stub();
 2365   return offset;
 2366 }
 2367 
 2368 // Emit deopt handler code.
 2369 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2370 {
 2371   // Note that the code buffer's insts_mark is always relative to insts.
 2372   // That's why we must use the macroassembler to generate a handler.
 2373   C2_MacroAssembler _masm(&cbuf);
 2374   address base = __ start_a_stub(size_deopt_handler());
 2375   if (base == NULL) {
 2376     ciEnv::current()->record_failure("CodeCache is full");
 2377     return 0;  // CodeBuffer::expand failed
 2378   }
 2379   int offset = __ offset();
 2380 
 2381   __ adr(lr, __ pc());
 2382   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2383 
 2384   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
 2385   __ end_a_stub();
 2386   return offset;
 2387 }
 2388 
 2389 // REQUIRED MATCHER CODE
 2390 
 2391 //=============================================================================
 2392 
 2393 const bool Matcher::match_rule_supported(int opcode) {
 2394   if (!has_match_rule(opcode))
 2395     return false;
 2396 
 2397   bool ret_value = true;
 2398   switch (opcode) {
 2399     case Op_CacheWB:
 2400     case Op_CacheWBPreSync:
 2401     case Op_CacheWBPostSync:
 2402       if (!VM_Version::supports_data_cache_line_flush()) {
 2403         ret_value = false;
 2404       }
 2405       break;
 2406     case Op_LoadVectorMasked:
 2407     case Op_StoreVectorMasked:
 2408     case Op_LoadVectorGatherMasked:
 2409     case Op_StoreVectorScatterMasked:
 2410     case Op_MaskAll:
 2411     case Op_AndVMask:
 2412     case Op_OrVMask:
 2413     case Op_XorVMask:
 2414       if (UseSVE == 0) {
 2415         ret_value = false;
 2416       }
 2417       break;
 2418   }
 2419 
 2420   return ret_value; // Per default match rules are supported.
 2421 }
 2422 
 2423 // Identify extra cases that we might want to provide match rules for vector nodes and
 2424 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2425 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2426   if (!match_rule_supported(opcode)) {
 2427     return false;
 2428   }
 2429   int bit_size = vlen * type2aelembytes(bt) * 8;
 2430   if (UseSVE == 0 && bit_size > 128) {
 2431     return false;
 2432   }
 2433   if (UseSVE > 0) {
 2434     return op_sve_supported(opcode, vlen, bt);
 2435   } else { // NEON
 2436     // Special cases
 2437     switch (opcode) {
 2438     case Op_VectorMaskCmp:
 2439     // We don't have VectorReinterpret with bit_size less than 64 support for
 2440     // now, even for byte type. To be refined with fully VectorCast support.
 2441     case Op_VectorReinterpret:
 2442       if (vlen < 2 || bit_size < 64) {
 2443         return false;
 2444       }
 2445       break;
 2446     case Op_MulAddVS2VI:
 2447       if (bit_size < 128) {
 2448         return false;
 2449       }
 2450       break;
 2451     case Op_MulVL:
 2452       return false;
 2453     case Op_VectorLoadShuffle:
 2454     case Op_VectorRearrange:
 2455       if (vlen < 4) {
 2456         return false;
 2457       }
 2458       break;
 2459     // Some types of VectorCast are not implemented for now.
 2460     case Op_VectorCastI2X:
 2461       if (bt == T_BYTE) {
 2462         return false;
 2463       }
 2464       break;
 2465     case Op_VectorCastS2X:
 2466       if (vlen < 4 || bit_size < 64) {
 2467         return false;
 2468       }
 2469       break;
 2470     case Op_VectorCastF2X:
 2471     case Op_VectorCastD2X:
 2472       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2473         return false;
 2474       }
 2475       break;
 2476     case Op_LoadVectorGather:
 2477     case Op_StoreVectorScatter:
 2478       return false;
 2479     default:
 2480       break;
 2481     }
 2482   }
 2483   return vector_size_supported(bt, vlen);
 2484 }
 2485 
 2486 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
 2487   // Only SVE supports masked operations.
 2488   if (UseSVE == 0) {
 2489     return false;
 2490   }
 2491   return match_rule_supported(opcode) &&
 2492          masked_op_sve_supported(opcode, vlen, bt);
 2493 }
 2494 
 2495 const RegMask* Matcher::predicate_reg_mask(void) {
 2496   return &_PR_REG_mask;
 2497 }
 2498 
 2499 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2500   return new TypeVectMask(elemTy, length);
 2501 }
 2502 
 2503 // Vector calling convention not yet implemented.
 2504 const bool Matcher::supports_vector_calling_convention(void) {
 2505   return false;
 2506 }
 2507 
 2508 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2509   Unimplemented();
 2510   return OptoRegPair(0, 0);
 2511 }
 2512 
 2513 // Is this branch offset short enough that a short branch can be used?
 2514 //
 2515 // NOTE: If the platform does not provide any short branch variants, then
 2516 //       this method should return false for offset 0.
 2517 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2518   // The passed offset is relative to address of the branch.
 2519 
 2520   return (-32768 <= offset && offset < 32768);
 2521 }
 2522 
 2523 // Vector width in bytes.
 2524 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2525   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2526   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2527   // Minimum 2 values in vector
 2528   if (size < 2*type2aelembytes(bt)) size = 0;
 2529   // But never < 4
 2530   if (size < 4) size = 0;
 2531   return size;
 2532 }
 2533 
 2534 // Limits on vector size (number of elements) loaded into vector.
 2535 const int Matcher::max_vector_size(const BasicType bt) {
 2536   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2537 }
 2538 
 2539 const int Matcher::min_vector_size(const BasicType bt) {
 2540   int max_size = max_vector_size(bt);
 2541   // Limit the min vector size to 8 bytes.
 2542   int size = 8 / type2aelembytes(bt);
 2543   if (bt == T_BYTE) {
 2544     // To support vector api shuffle/rearrange.
 2545     size = 4;
 2546   } else if (bt == T_BOOLEAN) {
 2547     // To support vector api load/store mask.
 2548     size = 2;
 2549   }
 2550   if (size < 2) size = 2;
 2551   return MIN2(size, max_size);
 2552 }
 2553 
 2554 // Actual max scalable vector register length.
 2555 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2556   return Matcher::max_vector_size(bt);
 2557 }
 2558 
 2559 // Vector ideal reg.
 2560 const uint Matcher::vector_ideal_reg(int len) {
 2561   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2562     return Op_VecA;
 2563   }
 2564   switch(len) {
 2565     // For 16-bit/32-bit mask vector, reuse VecD.
 2566     case  2:
 2567     case  4:
 2568     case  8: return Op_VecD;
 2569     case 16: return Op_VecX;
 2570   }
 2571   ShouldNotReachHere();
 2572   return 0;
 2573 }
 2574 
 2575 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2576   ShouldNotReachHere(); // generic vector operands not supported
 2577   return NULL;
 2578 }
 2579 
 2580 bool Matcher::is_reg2reg_move(MachNode* m) {
 2581   ShouldNotReachHere();  // generic vector operands not supported
 2582   return false;
 2583 }
 2584 
 2585 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2586   ShouldNotReachHere();  // generic vector operands not supported
 2587   return false;
 2588 }
 2589 
 2590 // Return whether or not this register is ever used as an argument.
 2591 // This function is used on startup to build the trampoline stubs in
 2592 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2593 // call in the trampoline, and arguments in those registers not be
 2594 // available to the callee.
 2595 bool Matcher::can_be_java_arg(int reg)
 2596 {
 2597   return
 2598     reg ==  R0_num || reg == R0_H_num ||
 2599     reg ==  R1_num || reg == R1_H_num ||
 2600     reg ==  R2_num || reg == R2_H_num ||
 2601     reg ==  R3_num || reg == R3_H_num ||
 2602     reg ==  R4_num || reg == R4_H_num ||
 2603     reg ==  R5_num || reg == R5_H_num ||
 2604     reg ==  R6_num || reg == R6_H_num ||
 2605     reg ==  R7_num || reg == R7_H_num ||
 2606     reg ==  V0_num || reg == V0_H_num ||
 2607     reg ==  V1_num || reg == V1_H_num ||
 2608     reg ==  V2_num || reg == V2_H_num ||
 2609     reg ==  V3_num || reg == V3_H_num ||
 2610     reg ==  V4_num || reg == V4_H_num ||
 2611     reg ==  V5_num || reg == V5_H_num ||
 2612     reg ==  V6_num || reg == V6_H_num ||
 2613     reg ==  V7_num || reg == V7_H_num;
 2614 }
 2615 
 2616 bool Matcher::is_spillable_arg(int reg)
 2617 {
 2618   return can_be_java_arg(reg);
 2619 }
 2620 
 2621 uint Matcher::int_pressure_limit()
 2622 {
 2623   // JDK-8183543: When taking the number of available registers as int
 2624   // register pressure threshold, the jtreg test:
 2625   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2626   // failed due to C2 compilation failure with
 2627   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2628   //
 2629   // A derived pointer is live at CallNode and then is flagged by RA
 2630   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2631   // derived pointers and lastly fail to spill after reaching maximum
 2632   // number of iterations. Lowering the default pressure threshold to
 2633   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2634   // a high register pressure area of the code so that split_DEF can
 2635   // generate DefinitionSpillCopy for the derived pointer.
 2636   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2637   if (!PreserveFramePointer) {
 2638     // When PreserveFramePointer is off, frame pointer is allocatable,
 2639     // but different from other SOC registers, it is excluded from
 2640     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2641     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2642     // See check_pressure_at_fatproj().
 2643     default_int_pressure_threshold--;
 2644   }
 2645   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2646 }
 2647 
 2648 uint Matcher::float_pressure_limit()
 2649 {
 2650   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2651   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2652 }
 2653 
 2654 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2655   return false;
 2656 }
 2657 
 2658 RegMask Matcher::divI_proj_mask() {
 2659   ShouldNotReachHere();
 2660   return RegMask();
 2661 }
 2662 
 2663 // Register for MODI projection of divmodI.
 2664 RegMask Matcher::modI_proj_mask() {
 2665   ShouldNotReachHere();
 2666   return RegMask();
 2667 }
 2668 
 2669 // Register for DIVL projection of divmodL.
 2670 RegMask Matcher::divL_proj_mask() {
 2671   ShouldNotReachHere();
 2672   return RegMask();
 2673 }
 2674 
 2675 // Register for MODL projection of divmodL.
 2676 RegMask Matcher::modL_proj_mask() {
 2677   ShouldNotReachHere();
 2678   return RegMask();
 2679 }
 2680 
 2681 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2682   return FP_REG_mask();
 2683 }
 2684 
 2685 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2686   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2687     Node* u = addp->fast_out(i);
 2688     if (u->is_LoadStore()) {
 2689       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2690       // instructions) only take register indirect as an operand, so
 2691       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2692       // must fail.
 2693       return false;
 2694     }
 2695     if (u->is_Mem()) {
 2696       int opsize = u->as_Mem()->memory_size();
 2697       assert(opsize > 0, "unexpected memory operand size");
 2698       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2699         return false;
 2700       }
 2701     }
 2702   }
 2703   return true;
 2704 }
 2705 
 2706 // Should the matcher clone input 'm' of node 'n'?
 2707 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2708   // ShiftV src (ShiftCntV con)
 2709   // StoreVector (VectorStoreMask src)
 2710   if (is_vshift_con_pattern(n, m) ||
 2711       (UseSVE > 0 && m->Opcode() == Op_VectorStoreMask && n->Opcode() == Op_StoreVector)) {
 2712     mstack.push(m, Visit);
 2713     return true;
 2714   }
 2715 
 2716   return false;
 2717 }
 2718 
 2719 // Should the Matcher clone shifts on addressing modes, expecting them
 2720 // to be subsumed into complex addressing expressions or compute them
 2721 // into registers?
 2722 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2723   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2724     return true;
 2725   }
 2726 
 2727   Node *off = m->in(AddPNode::Offset);
 2728   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2729       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2730       // Are there other uses besides address expressions?
 2731       !is_visited(off)) {
 2732     address_visited.set(off->_idx); // Flag as address_visited
 2733     mstack.push(off->in(2), Visit);
 2734     Node *conv = off->in(1);
 2735     if (conv->Opcode() == Op_ConvI2L &&
 2736         // Are there other uses besides address expressions?
 2737         !is_visited(conv)) {
 2738       address_visited.set(conv->_idx); // Flag as address_visited
 2739       mstack.push(conv->in(1), Pre_Visit);
 2740     } else {
 2741       mstack.push(conv, Pre_Visit);
 2742     }
 2743     address_visited.test_set(m->_idx); // Flag as address_visited
 2744     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2745     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2746     return true;
 2747   } else if (off->Opcode() == Op_ConvI2L &&
 2748              // Are there other uses besides address expressions?
 2749              !is_visited(off)) {
 2750     address_visited.test_set(m->_idx); // Flag as address_visited
 2751     address_visited.set(off->_idx); // Flag as address_visited
 2752     mstack.push(off->in(1), Pre_Visit);
 2753     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2754     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2755     return true;
 2756   }
 2757   return false;
 2758 }
 2759 
 2760 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2761   C2_MacroAssembler _masm(&cbuf);                                       \
 2762   {                                                                     \
 2763     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2764     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2765     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2766     __ INSN(REG, as_Register(BASE));                                    \
 2767   }
 2768 
 2769 
 2770 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2771   {
 2772     Address::extend scale;
 2773 
 2774     // Hooboy, this is fugly.  We need a way to communicate to the
 2775     // encoder that the index needs to be sign extended, so we have to
 2776     // enumerate all the cases.
 2777     switch (opcode) {
 2778     case INDINDEXSCALEDI2L:
 2779     case INDINDEXSCALEDI2LN:
 2780     case INDINDEXI2L:
 2781     case INDINDEXI2LN:
 2782       scale = Address::sxtw(size);
 2783       break;
 2784     default:
 2785       scale = Address::lsl(size);
 2786     }
 2787 
 2788     if (index == -1) {
 2789       return Address(base, disp);
 2790     } else {
 2791       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2792       return Address(base, as_Register(index), scale);
 2793     }
 2794   }
 2795 
 2796 
 2797 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2798 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2799 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2800 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2801                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2802 
 2803   // Used for all non-volatile memory accesses.  The use of
 2804   // $mem->opcode() to discover whether this pattern uses sign-extended
 2805   // offsets is something of a kludge.
 2806   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2807                         Register reg, int opcode,
 2808                         Register base, int index, int scale, int disp,
 2809                         int size_in_memory)
 2810   {
 2811     Address addr = mem2address(opcode, base, index, scale, disp);
 2812     if (addr.getMode() == Address::base_plus_offset) {
 2813       /* If we get an out-of-range offset it is a bug in the compiler,
 2814          so we assert here. */
 2815       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2816              "c2 compiler bug");
 2817       /* Fix up any out-of-range offsets. */
 2818       assert_different_registers(rscratch1, base);
 2819       assert_different_registers(rscratch1, reg);
 2820       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2821     }
 2822     (masm.*insn)(reg, addr);
 2823   }
 2824 
 2825   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2826                         FloatRegister reg, int opcode,
 2827                         Register base, int index, int size, int disp,
 2828                         int size_in_memory)
 2829   {
 2830     Address::extend scale;
 2831 
 2832     switch (opcode) {
 2833     case INDINDEXSCALEDI2L:
 2834     case INDINDEXSCALEDI2LN:
 2835       scale = Address::sxtw(size);
 2836       break;
 2837     default:
 2838       scale = Address::lsl(size);
 2839     }
 2840 
 2841     if (index == -1) {
 2842       /* If we get an out-of-range offset it is a bug in the compiler,
 2843          so we assert here. */
 2844       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2845       /* Fix up any out-of-range offsets. */
 2846       assert_different_registers(rscratch1, base);
 2847       Address addr = Address(base, disp);
 2848       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2849       (masm.*insn)(reg, addr);
 2850     } else {
 2851       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2852       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2853     }
 2854   }
 2855 
 2856   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2857                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2858                         int opcode, Register base, int index, int size, int disp)
 2859   {
 2860     if (index == -1) {
 2861       (masm.*insn)(reg, T, Address(base, disp));
 2862     } else {
 2863       assert(disp == 0, "unsupported address mode");
 2864       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2865     }
 2866   }
 2867 
 2868 %}
 2869 
 2870 
 2871 
 2872 //----------ENCODING BLOCK-----------------------------------------------------
 2873 // This block specifies the encoding classes used by the compiler to
 2874 // output byte streams.  Encoding classes are parameterized macros
 2875 // used by Machine Instruction Nodes in order to generate the bit
 2876 // encoding of the instruction.  Operands specify their base encoding
 2877 // interface with the interface keyword.  There are currently
 2878 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2879 // COND_INTER.  REG_INTER causes an operand to generate a function
 2880 // which returns its register number when queried.  CONST_INTER causes
 2881 // an operand to generate a function which returns the value of the
 2882 // constant when queried.  MEMORY_INTER causes an operand to generate
 2883 // four functions which return the Base Register, the Index Register,
 2884 // the Scale Value, and the Offset Value of the operand when queried.
 2885 // COND_INTER causes an operand to generate six functions which return
 2886 // the encoding code (ie - encoding bits for the instruction)
 2887 // associated with each basic boolean condition for a conditional
 2888 // instruction.
 2889 //
 2890 // Instructions specify two basic values for encoding.  Again, a
 2891 // function is available to check if the constant displacement is an
 2892 // oop. They use the ins_encode keyword to specify their encoding
 2893 // classes (which must be a sequence of enc_class names, and their
 2894 // parameters, specified in the encoding block), and they use the
 2895 // opcode keyword to specify, in order, their primary, secondary, and
 2896 // tertiary opcode.  Only the opcode sections which a particular
 2897 // instruction needs for encoding need to be specified.
 2898 encode %{
 2899   // Build emit functions for each basic byte or larger field in the
 2900   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2901   // from C++ code in the enc_class source block.  Emit functions will
 2902   // live in the main source block for now.  In future, we can
 2903   // generalize this by adding a syntax that specifies the sizes of
 2904   // fields in an order, so that the adlc can build the emit functions
 2905   // automagically
 2906 
 2907   // catch all for unimplemented encodings
 2908   enc_class enc_unimplemented %{
 2909     C2_MacroAssembler _masm(&cbuf);
 2910     __ unimplemented("C2 catch all");
 2911   %}
 2912 
 2913   // BEGIN Non-volatile memory access
 2914 
 2915   // This encoding class is generated automatically from ad_encode.m4.
 2916   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2917   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2918     Register dst_reg = as_Register($dst$$reg);
 2919     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2921   %}
 2922 
 2923   // This encoding class is generated automatically from ad_encode.m4.
 2924   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2925   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2926     Register dst_reg = as_Register($dst$$reg);
 2927     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2928                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2929   %}
 2930 
 2931   // This encoding class is generated automatically from ad_encode.m4.
 2932   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2933   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2934     Register dst_reg = as_Register($dst$$reg);
 2935     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2936                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2937   %}
 2938 
 2939   // This encoding class is generated automatically from ad_encode.m4.
 2940   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2941   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2942     Register dst_reg = as_Register($dst$$reg);
 2943     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2945   %}
 2946 
 2947   // This encoding class is generated automatically from ad_encode.m4.
 2948   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2949   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2950     Register dst_reg = as_Register($dst$$reg);
 2951     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2952                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2953   %}
 2954 
 2955   // This encoding class is generated automatically from ad_encode.m4.
 2956   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2957   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2958     Register dst_reg = as_Register($dst$$reg);
 2959     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2960                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2961   %}
 2962 
 2963   // This encoding class is generated automatically from ad_encode.m4.
 2964   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2965   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2966     Register dst_reg = as_Register($dst$$reg);
 2967     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2968                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2969   %}
 2970 
 2971   // This encoding class is generated automatically from ad_encode.m4.
 2972   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2973   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2974     Register dst_reg = as_Register($dst$$reg);
 2975     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2976                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2977   %}
 2978 
 2979   // This encoding class is generated automatically from ad_encode.m4.
 2980   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2981   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2982     Register dst_reg = as_Register($dst$$reg);
 2983     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2984                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2985   %}
 2986 
 2987   // This encoding class is generated automatically from ad_encode.m4.
 2988   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2989   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2990     Register dst_reg = as_Register($dst$$reg);
 2991     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2992                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2993   %}
 2994 
 2995   // This encoding class is generated automatically from ad_encode.m4.
 2996   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2997   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2998     Register dst_reg = as_Register($dst$$reg);
 2999     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 3000                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3001   %}
 3002 
 3003   // This encoding class is generated automatically from ad_encode.m4.
 3004   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3005   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 3006     Register dst_reg = as_Register($dst$$reg);
 3007     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 3008                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3009   %}
 3010 
 3011   // This encoding class is generated automatically from ad_encode.m4.
 3012   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3013   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3014     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3015     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3016                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3017   %}
 3018 
 3019   // This encoding class is generated automatically from ad_encode.m4.
 3020   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3021   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3022     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3023     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3024                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3025   %}
 3026 
 3027   // This encoding class is generated automatically from ad_encode.m4.
 3028   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3029   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3030     Register src_reg = as_Register($src$$reg);
 3031     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 3032                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3033   %}
 3034 
 3035   // This encoding class is generated automatically from ad_encode.m4.
 3036   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3037   enc_class aarch64_enc_strb0(memory1 mem) %{
 3038     C2_MacroAssembler _masm(&cbuf);
 3039     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3041   %}
 3042 
 3043   // This encoding class is generated automatically from ad_encode.m4.
 3044   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3045   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3046     Register src_reg = as_Register($src$$reg);
 3047     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3048                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3049   %}
 3050 
 3051   // This encoding class is generated automatically from ad_encode.m4.
 3052   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3053   enc_class aarch64_enc_strh0(memory2 mem) %{
 3054     C2_MacroAssembler _masm(&cbuf);
 3055     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3056                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3057   %}
 3058 
 3059   // This encoding class is generated automatically from ad_encode.m4.
 3060   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3061   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3062     Register src_reg = as_Register($src$$reg);
 3063     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3064                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3065   %}
 3066 
 3067   // This encoding class is generated automatically from ad_encode.m4.
 3068   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3069   enc_class aarch64_enc_strw0(memory4 mem) %{
 3070     C2_MacroAssembler _masm(&cbuf);
 3071     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3072                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3073   %}
 3074 
 3075   // This encoding class is generated automatically from ad_encode.m4.
 3076   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3077   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3078     Register src_reg = as_Register($src$$reg);
 3079     // we sometimes get asked to store the stack pointer into the
 3080     // current thread -- we cannot do that directly on AArch64
 3081     if (src_reg == r31_sp) {
 3082       C2_MacroAssembler _masm(&cbuf);
 3083       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3084       __ mov(rscratch2, sp);
 3085       src_reg = rscratch2;
 3086     }
 3087     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3088                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3089   %}
 3090 
 3091   // This encoding class is generated automatically from ad_encode.m4.
 3092   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3093   enc_class aarch64_enc_str0(memory8 mem) %{
 3094     C2_MacroAssembler _masm(&cbuf);
 3095     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3096                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3097   %}
 3098 
 3099   // This encoding class is generated automatically from ad_encode.m4.
 3100   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3101   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3102     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3103     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3104                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3105   %}
 3106 
 3107   // This encoding class is generated automatically from ad_encode.m4.
 3108   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3109   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3110     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3111     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3112                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3113   %}
 3114 
 3115   // This encoding class is generated automatically from ad_encode.m4.
 3116   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3117   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3118       C2_MacroAssembler _masm(&cbuf);
 3119       __ membar(Assembler::StoreStore);
 3120       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3121                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3122   %}
 3123 
 3124   // END Non-volatile memory access
 3125 
 3126   // Vector loads and stores
 3127   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3128     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3129     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3130        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3134     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3135     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3136        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3137   %}
 3138 
 3139   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3140     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3141     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3142        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3143   %}
 3144 
 3145   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3146     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3147     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3148        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3149   %}
 3150 
 3151   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3152     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3153     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3154        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3155   %}
 3156 
 3157   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3158     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3159     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3160        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3161   %}
 3162 
 3163   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3164     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3165     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3166        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3167   %}
 3168 
 3169   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3170     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3171     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3172        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3173   %}
 3174 
 3175   // volatile loads and stores
 3176 
 3177   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3178     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3179                  rscratch1, stlrb);
 3180   %}
 3181 
 3182   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3183     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3184                  rscratch1, stlrh);
 3185   %}
 3186 
 3187   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3188     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3189                  rscratch1, stlrw);
 3190   %}
 3191 
 3192 
 3193   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3194     Register dst_reg = as_Register($dst$$reg);
 3195     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3196              rscratch1, ldarb);
 3197     __ sxtbw(dst_reg, dst_reg);
 3198   %}
 3199 
 3200   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3201     Register dst_reg = as_Register($dst$$reg);
 3202     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3203              rscratch1, ldarb);
 3204     __ sxtb(dst_reg, dst_reg);
 3205   %}
 3206 
 3207   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3208     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3209              rscratch1, ldarb);
 3210   %}
 3211 
 3212   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3213     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3214              rscratch1, ldarb);
 3215   %}
 3216 
 3217   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3218     Register dst_reg = as_Register($dst$$reg);
 3219     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3220              rscratch1, ldarh);
 3221     __ sxthw(dst_reg, dst_reg);
 3222   %}
 3223 
 3224   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3225     Register dst_reg = as_Register($dst$$reg);
 3226     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3227              rscratch1, ldarh);
 3228     __ sxth(dst_reg, dst_reg);
 3229   %}
 3230 
 3231   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3232     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3233              rscratch1, ldarh);
 3234   %}
 3235 
 3236   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3237     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3238              rscratch1, ldarh);
 3239   %}
 3240 
 3241   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3242     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3243              rscratch1, ldarw);
 3244   %}
 3245 
 3246   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3247     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3248              rscratch1, ldarw);
 3249   %}
 3250 
 3251   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3252     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3253              rscratch1, ldar);
 3254   %}
 3255 
 3256   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3257     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3258              rscratch1, ldarw);
 3259     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3260   %}
 3261 
 3262   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3263     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3264              rscratch1, ldar);
 3265     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3266   %}
 3267 
 3268   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3269     Register src_reg = as_Register($src$$reg);
 3270     // we sometimes get asked to store the stack pointer into the
 3271     // current thread -- we cannot do that directly on AArch64
 3272     if (src_reg == r31_sp) {
 3273       C2_MacroAssembler _masm(&cbuf);
 3274       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3275       __ mov(rscratch2, sp);
 3276       src_reg = rscratch2;
 3277     }
 3278     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3279                  rscratch1, stlr);
 3280   %}
 3281 
 3282   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3283     {
 3284       C2_MacroAssembler _masm(&cbuf);
 3285       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3286       __ fmovs(rscratch2, src_reg);
 3287     }
 3288     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3289                  rscratch1, stlrw);
 3290   %}
 3291 
 3292   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3293     {
 3294       C2_MacroAssembler _masm(&cbuf);
 3295       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3296       __ fmovd(rscratch2, src_reg);
 3297     }
 3298     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3299                  rscratch1, stlr);
 3300   %}
 3301 
 3302   // synchronized read/update encodings
 3303 
 3304   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3305     C2_MacroAssembler _masm(&cbuf);
 3306     Register dst_reg = as_Register($dst$$reg);
 3307     Register base = as_Register($mem$$base);
 3308     int index = $mem$$index;
 3309     int scale = $mem$$scale;
 3310     int disp = $mem$$disp;
 3311     if (index == -1) {
 3312        if (disp != 0) {
 3313         __ lea(rscratch1, Address(base, disp));
 3314         __ ldaxr(dst_reg, rscratch1);
 3315       } else {
 3316         // TODO
 3317         // should we ever get anything other than this case?
 3318         __ ldaxr(dst_reg, base);
 3319       }
 3320     } else {
 3321       Register index_reg = as_Register(index);
 3322       if (disp == 0) {
 3323         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3324         __ ldaxr(dst_reg, rscratch1);
 3325       } else {
 3326         __ lea(rscratch1, Address(base, disp));
 3327         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3328         __ ldaxr(dst_reg, rscratch1);
 3329       }
 3330     }
 3331   %}
 3332 
 3333   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3334     C2_MacroAssembler _masm(&cbuf);
 3335     Register src_reg = as_Register($src$$reg);
 3336     Register base = as_Register($mem$$base);
 3337     int index = $mem$$index;
 3338     int scale = $mem$$scale;
 3339     int disp = $mem$$disp;
 3340     if (index == -1) {
 3341        if (disp != 0) {
 3342         __ lea(rscratch2, Address(base, disp));
 3343         __ stlxr(rscratch1, src_reg, rscratch2);
 3344       } else {
 3345         // TODO
 3346         // should we ever get anything other than this case?
 3347         __ stlxr(rscratch1, src_reg, base);
 3348       }
 3349     } else {
 3350       Register index_reg = as_Register(index);
 3351       if (disp == 0) {
 3352         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3353         __ stlxr(rscratch1, src_reg, rscratch2);
 3354       } else {
 3355         __ lea(rscratch2, Address(base, disp));
 3356         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3357         __ stlxr(rscratch1, src_reg, rscratch2);
 3358       }
 3359     }
 3360     __ cmpw(rscratch1, zr);
 3361   %}
 3362 
 3363   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3364     C2_MacroAssembler _masm(&cbuf);
 3365     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3366     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3367                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3368                /*weak*/ false, noreg);
 3369   %}
 3370 
 3371   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3372     C2_MacroAssembler _masm(&cbuf);
 3373     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3374     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3375                Assembler::word, /*acquire*/ false, /*release*/ true,
 3376                /*weak*/ false, noreg);
 3377   %}
 3378 
 3379   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3380     C2_MacroAssembler _masm(&cbuf);
 3381     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3382     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3383                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3384                /*weak*/ false, noreg);
 3385   %}
 3386 
 3387   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3388     C2_MacroAssembler _masm(&cbuf);
 3389     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3390     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3391                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3392                /*weak*/ false, noreg);
 3393   %}
 3394 
 3395 
 3396   // The only difference between aarch64_enc_cmpxchg and
 3397   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3398   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3399   // lock.
 3400   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3401     C2_MacroAssembler _masm(&cbuf);
 3402     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3403     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3404                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3405                /*weak*/ false, noreg);
 3406   %}
 3407 
 3408   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3409     C2_MacroAssembler _masm(&cbuf);
 3410     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3411     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3412                Assembler::word, /*acquire*/ true, /*release*/ true,
 3413                /*weak*/ false, noreg);
 3414   %}
 3415 
 3416   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3417     C2_MacroAssembler _masm(&cbuf);
 3418     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3419     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3420                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3421                /*weak*/ false, noreg);
 3422   %}
 3423 
 3424   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3425     C2_MacroAssembler _masm(&cbuf);
 3426     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3427     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3428                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3429                /*weak*/ false, noreg);
 3430   %}
 3431 
 3432   // auxiliary used for CompareAndSwapX to set result register
 3433   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3434     C2_MacroAssembler _masm(&cbuf);
 3435     Register res_reg = as_Register($res$$reg);
 3436     __ cset(res_reg, Assembler::EQ);
 3437   %}
 3438 
 3439   // prefetch encodings
 3440 
 3441   enc_class aarch64_enc_prefetchw(memory mem) %{
 3442     C2_MacroAssembler _masm(&cbuf);
 3443     Register base = as_Register($mem$$base);
 3444     int index = $mem$$index;
 3445     int scale = $mem$$scale;
 3446     int disp = $mem$$disp;
 3447     if (index == -1) {
 3448       __ prfm(Address(base, disp), PSTL1KEEP);
 3449     } else {
 3450       Register index_reg = as_Register(index);
 3451       if (disp == 0) {
 3452         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3453       } else {
 3454         __ lea(rscratch1, Address(base, disp));
 3455 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3456       }
 3457     }
 3458   %}
 3459 
 3460   /// mov envcodings
 3461 
 3462   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3463     C2_MacroAssembler _masm(&cbuf);
 3464     uint32_t con = (uint32_t)$src$$constant;
 3465     Register dst_reg = as_Register($dst$$reg);
 3466     if (con == 0) {
 3467       __ movw(dst_reg, zr);
 3468     } else {
 3469       __ movw(dst_reg, con);
 3470     }
 3471   %}
 3472 
 3473   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3474     C2_MacroAssembler _masm(&cbuf);
 3475     Register dst_reg = as_Register($dst$$reg);
 3476     uint64_t con = (uint64_t)$src$$constant;
 3477     if (con == 0) {
 3478       __ mov(dst_reg, zr);
 3479     } else {
 3480       __ mov(dst_reg, con);
 3481     }
 3482   %}
 3483 
 3484   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3485     C2_MacroAssembler _masm(&cbuf);
 3486     Register dst_reg = as_Register($dst$$reg);
 3487     address con = (address)$src$$constant;
 3488     if (con == NULL || con == (address)1) {
 3489       ShouldNotReachHere();
 3490     } else {
 3491       relocInfo::relocType rtype = $src->constant_reloc();
 3492       if (rtype == relocInfo::oop_type) {
 3493         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3494       } else if (rtype == relocInfo::metadata_type) {
 3495         __ mov_metadata(dst_reg, (Metadata*)con);
 3496       } else {
 3497         assert(rtype == relocInfo::none, "unexpected reloc type");
 3498         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3499           __ mov(dst_reg, con);
 3500         } else {
 3501           uint64_t offset;
 3502           __ adrp(dst_reg, con, offset);
 3503           __ add(dst_reg, dst_reg, offset);
 3504         }
 3505       }
 3506     }
 3507   %}
 3508 
 3509   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3510     C2_MacroAssembler _masm(&cbuf);
 3511     Register dst_reg = as_Register($dst$$reg);
 3512     __ mov(dst_reg, zr);
 3513   %}
 3514 
 3515   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3516     C2_MacroAssembler _masm(&cbuf);
 3517     Register dst_reg = as_Register($dst$$reg);
 3518     __ mov(dst_reg, (uint64_t)1);
 3519   %}
 3520 
 3521   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3522     C2_MacroAssembler _masm(&cbuf);
 3523     __ load_byte_map_base($dst$$Register);
 3524   %}
 3525 
 3526   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3527     C2_MacroAssembler _masm(&cbuf);
 3528     Register dst_reg = as_Register($dst$$reg);
 3529     address con = (address)$src$$constant;
 3530     if (con == NULL) {
 3531       ShouldNotReachHere();
 3532     } else {
 3533       relocInfo::relocType rtype = $src->constant_reloc();
 3534       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3535       __ set_narrow_oop(dst_reg, (jobject)con);
 3536     }
 3537   %}
 3538 
 3539   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3540     C2_MacroAssembler _masm(&cbuf);
 3541     Register dst_reg = as_Register($dst$$reg);
 3542     __ mov(dst_reg, zr);
 3543   %}
 3544 
 3545   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3546     C2_MacroAssembler _masm(&cbuf);
 3547     Register dst_reg = as_Register($dst$$reg);
 3548     address con = (address)$src$$constant;
 3549     if (con == NULL) {
 3550       ShouldNotReachHere();
 3551     } else {
 3552       relocInfo::relocType rtype = $src->constant_reloc();
 3553       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3554       __ set_narrow_klass(dst_reg, (Klass *)con);
 3555     }
 3556   %}
 3557 
 3558   // arithmetic encodings
 3559 
 3560   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3561     C2_MacroAssembler _masm(&cbuf);
 3562     Register dst_reg = as_Register($dst$$reg);
 3563     Register src_reg = as_Register($src1$$reg);
 3564     int32_t con = (int32_t)$src2$$constant;
 3565     // add has primary == 0, subtract has primary == 1
 3566     if ($primary) { con = -con; }
 3567     if (con < 0) {
 3568       __ subw(dst_reg, src_reg, -con);
 3569     } else {
 3570       __ addw(dst_reg, src_reg, con);
 3571     }
 3572   %}
 3573 
 3574   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3575     C2_MacroAssembler _masm(&cbuf);
 3576     Register dst_reg = as_Register($dst$$reg);
 3577     Register src_reg = as_Register($src1$$reg);
 3578     int32_t con = (int32_t)$src2$$constant;
 3579     // add has primary == 0, subtract has primary == 1
 3580     if ($primary) { con = -con; }
 3581     if (con < 0) {
 3582       __ sub(dst_reg, src_reg, -con);
 3583     } else {
 3584       __ add(dst_reg, src_reg, con);
 3585     }
 3586   %}
 3587 
 3588   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3589     C2_MacroAssembler _masm(&cbuf);
 3590    Register dst_reg = as_Register($dst$$reg);
 3591    Register src1_reg = as_Register($src1$$reg);
 3592    Register src2_reg = as_Register($src2$$reg);
 3593     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3594   %}
 3595 
 3596   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3597     C2_MacroAssembler _masm(&cbuf);
 3598    Register dst_reg = as_Register($dst$$reg);
 3599    Register src1_reg = as_Register($src1$$reg);
 3600    Register src2_reg = as_Register($src2$$reg);
 3601     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3602   %}
 3603 
 3604   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3605     C2_MacroAssembler _masm(&cbuf);
 3606    Register dst_reg = as_Register($dst$$reg);
 3607    Register src1_reg = as_Register($src1$$reg);
 3608    Register src2_reg = as_Register($src2$$reg);
 3609     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3610   %}
 3611 
 3612   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3613     C2_MacroAssembler _masm(&cbuf);
 3614    Register dst_reg = as_Register($dst$$reg);
 3615    Register src1_reg = as_Register($src1$$reg);
 3616    Register src2_reg = as_Register($src2$$reg);
 3617     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3618   %}
 3619 
 3620   // compare instruction encodings
 3621 
 3622   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3623     C2_MacroAssembler _masm(&cbuf);
 3624     Register reg1 = as_Register($src1$$reg);
 3625     Register reg2 = as_Register($src2$$reg);
 3626     __ cmpw(reg1, reg2);
 3627   %}
 3628 
 3629   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3630     C2_MacroAssembler _masm(&cbuf);
 3631     Register reg = as_Register($src1$$reg);
 3632     int32_t val = $src2$$constant;
 3633     if (val >= 0) {
 3634       __ subsw(zr, reg, val);
 3635     } else {
 3636       __ addsw(zr, reg, -val);
 3637     }
 3638   %}
 3639 
 3640   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3641     C2_MacroAssembler _masm(&cbuf);
 3642     Register reg1 = as_Register($src1$$reg);
 3643     uint32_t val = (uint32_t)$src2$$constant;
 3644     __ movw(rscratch1, val);
 3645     __ cmpw(reg1, rscratch1);
 3646   %}
 3647 
 3648   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3649     C2_MacroAssembler _masm(&cbuf);
 3650     Register reg1 = as_Register($src1$$reg);
 3651     Register reg2 = as_Register($src2$$reg);
 3652     __ cmp(reg1, reg2);
 3653   %}
 3654 
 3655   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3656     C2_MacroAssembler _masm(&cbuf);
 3657     Register reg = as_Register($src1$$reg);
 3658     int64_t val = $src2$$constant;
 3659     if (val >= 0) {
 3660       __ subs(zr, reg, val);
 3661     } else if (val != -val) {
 3662       __ adds(zr, reg, -val);
 3663     } else {
 3664     // aargh, Long.MIN_VALUE is a special case
 3665       __ orr(rscratch1, zr, (uint64_t)val);
 3666       __ subs(zr, reg, rscratch1);
 3667     }
 3668   %}
 3669 
 3670   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3671     C2_MacroAssembler _masm(&cbuf);
 3672     Register reg1 = as_Register($src1$$reg);
 3673     uint64_t val = (uint64_t)$src2$$constant;
 3674     __ mov(rscratch1, val);
 3675     __ cmp(reg1, rscratch1);
 3676   %}
 3677 
 3678   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3679     C2_MacroAssembler _masm(&cbuf);
 3680     Register reg1 = as_Register($src1$$reg);
 3681     Register reg2 = as_Register($src2$$reg);
 3682     __ cmp(reg1, reg2);
 3683   %}
 3684 
 3685   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3686     C2_MacroAssembler _masm(&cbuf);
 3687     Register reg1 = as_Register($src1$$reg);
 3688     Register reg2 = as_Register($src2$$reg);
 3689     __ cmpw(reg1, reg2);
 3690   %}
 3691 
 3692   enc_class aarch64_enc_testp(iRegP src) %{
 3693     C2_MacroAssembler _masm(&cbuf);
 3694     Register reg = as_Register($src$$reg);
 3695     __ cmp(reg, zr);
 3696   %}
 3697 
 3698   enc_class aarch64_enc_testn(iRegN src) %{
 3699     C2_MacroAssembler _masm(&cbuf);
 3700     Register reg = as_Register($src$$reg);
 3701     __ cmpw(reg, zr);
 3702   %}
 3703 
 3704   enc_class aarch64_enc_b(label lbl) %{
 3705     C2_MacroAssembler _masm(&cbuf);
 3706     Label *L = $lbl$$label;
 3707     __ b(*L);
 3708   %}
 3709 
 3710   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3711     C2_MacroAssembler _masm(&cbuf);
 3712     Label *L = $lbl$$label;
 3713     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3714   %}
 3715 
 3716   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3717     C2_MacroAssembler _masm(&cbuf);
 3718     Label *L = $lbl$$label;
 3719     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3720   %}
 3721 
 3722   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3723   %{
 3724      Register sub_reg = as_Register($sub$$reg);
 3725      Register super_reg = as_Register($super$$reg);
 3726      Register temp_reg = as_Register($temp$$reg);
 3727      Register result_reg = as_Register($result$$reg);
 3728 
 3729      Label miss;
 3730      C2_MacroAssembler _masm(&cbuf);
 3731      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3732                                      NULL, &miss,
 3733                                      /*set_cond_codes:*/ true);
 3734      if ($primary) {
 3735        __ mov(result_reg, zr);
 3736      }
 3737      __ bind(miss);
 3738   %}
 3739 
 3740   enc_class aarch64_enc_java_static_call(method meth) %{
 3741     C2_MacroAssembler _masm(&cbuf);
 3742 
 3743     address addr = (address)$meth$$method;
 3744     address call;
 3745     if (!_method) {
 3746       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3747       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3748       if (call == NULL) {
 3749         ciEnv::current()->record_failure("CodeCache is full");
 3750         return;
 3751       }
 3752     } else {
 3753       int method_index = resolved_method_index(cbuf);
 3754       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3755                                                   : static_call_Relocation::spec(method_index);
 3756       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3757       if (call == NULL) {
 3758         ciEnv::current()->record_failure("CodeCache is full");
 3759         return;
 3760       }
 3761       // Emit stub for static call
 3762       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3763       if (stub == NULL) {
 3764         ciEnv::current()->record_failure("CodeCache is full");
 3765         return;
 3766       }
 3767     }
 3768 
 3769     // Only non uncommon_trap calls need to reinitialize ptrue.
 3770     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3771       __ reinitialize_ptrue();
 3772     }
 3773   %}
 3774 
 3775   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3776     C2_MacroAssembler _masm(&cbuf);
 3777     int method_index = resolved_method_index(cbuf);
 3778     address call = __ ic_call((address)$meth$$method, method_index);
 3779     if (call == NULL) {
 3780       ciEnv::current()->record_failure("CodeCache is full");
 3781       return;
 3782     } else if (Compile::current()->max_vector_size() > 0) {
 3783       __ reinitialize_ptrue();
 3784     }
 3785   %}
 3786 
 3787   enc_class aarch64_enc_call_epilog() %{
 3788     C2_MacroAssembler _masm(&cbuf);
 3789     if (VerifyStackAtCalls) {
 3790       // Check that stack depth is unchanged: find majik cookie on stack
 3791       __ call_Unimplemented();
 3792     }
 3793   %}
 3794 
 3795   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3796     C2_MacroAssembler _masm(&cbuf);
 3797 
 3798     // some calls to generated routines (arraycopy code) are scheduled
 3799     // by C2 as runtime calls. if so we can call them using a br (they
 3800     // will be in a reachable segment) otherwise we have to use a blr
 3801     // which loads the absolute address into a register.
 3802     address entry = (address)$meth$$method;
 3803     CodeBlob *cb = CodeCache::find_blob(entry);
 3804     if (cb) {
 3805       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3806       if (call == NULL) {
 3807         ciEnv::current()->record_failure("CodeCache is full");
 3808         return;
 3809       }
 3810     } else {
 3811       Label retaddr;
 3812       __ adr(rscratch2, retaddr);
 3813       __ lea(rscratch1, RuntimeAddress(entry));
 3814       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3815       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3816       __ blr(rscratch1);
 3817       __ bind(retaddr);
 3818       __ add(sp, sp, 2 * wordSize);
 3819     }
 3820     if (Compile::current()->max_vector_size() > 0) {
 3821       __ reinitialize_ptrue();
 3822     }
 3823   %}
 3824 
 3825   enc_class aarch64_enc_rethrow() %{
 3826     C2_MacroAssembler _masm(&cbuf);
 3827     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3828   %}
 3829 
 3830   enc_class aarch64_enc_ret() %{
 3831     C2_MacroAssembler _masm(&cbuf);
 3832 #ifdef ASSERT
 3833     if (Compile::current()->max_vector_size() > 0) {
 3834       __ verify_ptrue();
 3835     }
 3836 #endif
 3837     __ ret(lr);
 3838   %}
 3839 
 3840   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3841     C2_MacroAssembler _masm(&cbuf);
 3842     Register target_reg = as_Register($jump_target$$reg);
 3843     __ br(target_reg);
 3844   %}
 3845 
 3846   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3847     C2_MacroAssembler _masm(&cbuf);
 3848     Register target_reg = as_Register($jump_target$$reg);
 3849     // exception oop should be in r0
 3850     // ret addr has been popped into lr
 3851     // callee expects it in r3
 3852     __ mov(r3, lr);
 3853     __ br(target_reg);
 3854   %}
 3855 
 3856   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3857     C2_MacroAssembler _masm(&cbuf);
 3858     Register oop = as_Register($object$$reg);
 3859     Register box = as_Register($box$$reg);
 3860     Register disp_hdr = as_Register($tmp$$reg);
 3861     Register tmp = as_Register($tmp2$$reg);
 3862     Label cont;
 3863     Label object_has_monitor;
 3864     Label cas_failed;
 3865 
 3866     assert_different_registers(oop, box, tmp, disp_hdr);
 3867 
 3868     // Load markWord from object into displaced_header.
 3869     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3870 
 3871     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3872       __ load_klass(tmp, oop);
 3873       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3874       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3875       __ br(Assembler::NE, cont);
 3876     }
 3877 
 3878     // Check for existing monitor
 3879     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3880 
 3881     // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3882     __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3883 
 3884     // Initialize the box. (Must happen before we update the object mark!)
 3885     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3886 
 3887     // Compare object markWord with an unlocked value (tmp) and if
 3888     // equal exchange the stack address of our box with object markWord.
 3889     // On failure disp_hdr contains the possibly locked markWord.
 3890     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3891                /*release*/ true, /*weak*/ false, disp_hdr);
 3892     __ br(Assembler::EQ, cont);
 3893 
 3894     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3895 
 3896     // If the compare-and-exchange succeeded, then we found an unlocked
 3897     // object, will have now locked it will continue at label cont
 3898 
 3899     __ bind(cas_failed);
 3900     // We did not see an unlocked object so try the fast recursive case.
 3901 
 3902     // Check if the owner is self by comparing the value in the
 3903     // markWord of object (disp_hdr) with the stack pointer.
 3904     __ mov(rscratch1, sp);
 3905     __ sub(disp_hdr, disp_hdr, rscratch1);
 3906     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3907     // If condition is true we are cont and hence we can store 0 as the
 3908     // displaced header in the box, which indicates that it is a recursive lock.
 3909     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3910     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3911 
 3912     __ b(cont);
 3913 
 3914     // Handle existing monitor.
 3915     __ bind(object_has_monitor);
 3916 
 3917     // The object's monitor m is unlocked iff m->owner == NULL,
 3918     // otherwise m->owner may contain a thread or a stack address.
 3919     //
 3920     // Try to CAS m->owner from NULL to current thread.
 3921     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3922     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3923                /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
 3924 
 3925     // Store a non-null value into the box to avoid looking like a re-entrant
 3926     // lock. The fast-path monitor unlock code checks for
 3927     // markWord::monitor_value so use markWord::unused_mark which has the
 3928     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3929     __ mov(tmp, (address)markWord::unused_mark().value());
 3930     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3931 
 3932     __ bind(cont);
 3933     // flag == EQ indicates success
 3934     // flag == NE indicates failure
 3935   %}
 3936 
 3937   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3938     C2_MacroAssembler _masm(&cbuf);
 3939     Register oop = as_Register($object$$reg);
 3940     Register box = as_Register($box$$reg);
 3941     Register disp_hdr = as_Register($tmp$$reg);
 3942     Register tmp = as_Register($tmp2$$reg);
 3943     Label cont;
 3944     Label object_has_monitor;
 3945 
 3946     assert_different_registers(oop, box, tmp, disp_hdr);
 3947 
 3948     // Find the lock address and load the displaced header from the stack.
 3949     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3950 
 3951     // If the displaced header is 0, we have a recursive unlock.
 3952     __ cmp(disp_hdr, zr);
 3953     __ br(Assembler::EQ, cont);
 3954 
 3955     // Handle existing monitor.
 3956     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3957     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3958 
 3959     // Check if it is still a light weight lock, this is is true if we
 3960     // see the stack address of the basicLock in the markWord of the
 3961     // object.
 3962 
 3963     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3964                /*release*/ true, /*weak*/ false, tmp);
 3965     __ b(cont);
 3966 
 3967     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3968 
 3969     // Handle existing monitor.
 3970     __ bind(object_has_monitor);
 3971     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3972     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3973     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3974     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3975     __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
 3976     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
 3977     __ cmp(rscratch1, zr); // Sets flags for result
 3978     __ br(Assembler::NE, cont);
 3979 
 3980     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3981     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3982     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3983     __ cmp(rscratch1, zr); // Sets flags for result
 3984     __ cbnz(rscratch1, cont);
 3985     // need a release store here
 3986     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3987     __ stlr(zr, tmp); // set unowned
 3988 
 3989     __ bind(cont);
 3990     // flag == EQ indicates success
 3991     // flag == NE indicates failure
 3992   %}
 3993 
 3994 %}
 3995 
 3996 //----------FRAME--------------------------------------------------------------
 3997 // Definition of frame structure and management information.
 3998 //
 3999 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4000 //                             |   (to get allocators register number
 4001 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4002 //  r   CALLER     |        |
 4003 //  o     |        +--------+      pad to even-align allocators stack-slot
 4004 //  w     V        |  pad0  |        numbers; owned by CALLER
 4005 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4006 //  h     ^        |   in   |  5
 4007 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4008 //  |     |        |        |  3
 4009 //  |     |        +--------+
 4010 //  V     |        | old out|      Empty on Intel, window on Sparc
 4011 //        |    old |preserve|      Must be even aligned.
 4012 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4013 //        |        |   in   |  3   area for Intel ret address
 4014 //     Owned by    |preserve|      Empty on Sparc.
 4015 //       SELF      +--------+
 4016 //        |        |  pad2  |  2   pad to align old SP
 4017 //        |        +--------+  1
 4018 //        |        | locks  |  0
 4019 //        |        +--------+----> OptoReg::stack0(), even aligned
 4020 //        |        |  pad1  | 11   pad to align new SP
 4021 //        |        +--------+
 4022 //        |        |        | 10
 4023 //        |        | spills |  9   spills
 4024 //        V        |        |  8   (pad0 slot for callee)
 4025 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4026 //        ^        |  out   |  7
 4027 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4028 //     Owned by    +--------+
 4029 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4030 //        |    new |preserve|      Must be even-aligned.
 4031 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4032 //        |        |        |
 4033 //
 4034 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4035 //         known from SELF's arguments and the Java calling convention.
 4036 //         Region 6-7 is determined per call site.
 4037 // Note 2: If the calling convention leaves holes in the incoming argument
 4038 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4039 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4040 //         incoming area, as the Java calling convention is completely under
 4041 //         the control of the AD file.  Doubles can be sorted and packed to
 4042 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4043 //         varargs C calling conventions.
 4044 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4045 //         even aligned with pad0 as needed.
 4046 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4047 //           (the latter is true on Intel but is it false on AArch64?)
 4048 //         region 6-11 is even aligned; it may be padded out more so that
 4049 //         the region from SP to FP meets the minimum stack alignment.
 4050 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4051 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4052 //         SP meets the minimum alignment.
 4053 
 4054 frame %{
 4055   // These three registers define part of the calling convention
 4056   // between compiled code and the interpreter.
 4057 
 4058   // Inline Cache Register or Method for I2C.
 4059   inline_cache_reg(R12);
 4060 
 4061   // Number of stack slots consumed by locking an object
 4062   sync_stack_slots(2);
 4063 
 4064   // Compiled code's Frame Pointer
 4065   frame_pointer(R31);
 4066 
 4067   // Interpreter stores its frame pointer in a register which is
 4068   // stored to the stack by I2CAdaptors.
 4069   // I2CAdaptors convert from interpreted java to compiled java.
 4070   interpreter_frame_pointer(R29);
 4071 
 4072   // Stack alignment requirement
 4073   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4074 
 4075   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4076   // for calls to C.  Supports the var-args backing area for register parms.
 4077   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4078 
 4079   // The after-PROLOG location of the return address.  Location of
 4080   // return address specifies a type (REG or STACK) and a number
 4081   // representing the register number (i.e. - use a register name) or
 4082   // stack slot.
 4083   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4084   // Otherwise, it is above the locks and verification slot and alignment word
 4085   // TODO this may well be correct but need to check why that - 2 is there
 4086   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4087   // which folds in the space used for monitors
 4088   return_addr(STACK - 2 +
 4089               align_up((Compile::current()->in_preserve_stack_slots() +
 4090                         Compile::current()->fixed_slots()),
 4091                        stack_alignment_in_slots()));
 4092 
 4093   // Location of compiled Java return values.  Same as C for now.
 4094   return_value
 4095   %{
 4096     // TODO do we allow ideal_reg == Op_RegN???
 4097     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4098            "only return normal values");
 4099 
 4100     static const int lo[Op_RegL + 1] = { // enum name
 4101       0,                                 // Op_Node
 4102       0,                                 // Op_Set
 4103       R0_num,                            // Op_RegN
 4104       R0_num,                            // Op_RegI
 4105       R0_num,                            // Op_RegP
 4106       V0_num,                            // Op_RegF
 4107       V0_num,                            // Op_RegD
 4108       R0_num                             // Op_RegL
 4109     };
 4110 
 4111     static const int hi[Op_RegL + 1] = { // enum name
 4112       0,                                 // Op_Node
 4113       0,                                 // Op_Set
 4114       OptoReg::Bad,                      // Op_RegN
 4115       OptoReg::Bad,                      // Op_RegI
 4116       R0_H_num,                          // Op_RegP
 4117       OptoReg::Bad,                      // Op_RegF
 4118       V0_H_num,                          // Op_RegD
 4119       R0_H_num                           // Op_RegL
 4120     };
 4121 
 4122     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4123   %}
 4124 %}
 4125 
 4126 //----------ATTRIBUTES---------------------------------------------------------
 4127 //----------Operand Attributes-------------------------------------------------
 4128 op_attrib op_cost(1);        // Required cost attribute
 4129 
 4130 //----------Instruction Attributes---------------------------------------------
 4131 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4132 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4133 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4134                                 // a non-matching short branch variant
 4135                                 // of some long branch?
 4136 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4137                                 // be a power of 2) specifies the
 4138                                 // alignment that some part of the
 4139                                 // instruction (not necessarily the
 4140                                 // start) requires.  If > 1, a
 4141                                 // compute_padding() function must be
 4142                                 // provided for the instruction
 4143 
 4144 //----------OPERANDS-----------------------------------------------------------
 4145 // Operand definitions must precede instruction definitions for correct parsing
 4146 // in the ADLC because operands constitute user defined types which are used in
 4147 // instruction definitions.
 4148 
 4149 //----------Simple Operands----------------------------------------------------
 4150 
 4151 // Integer operands 32 bit
 4152 // 32 bit immediate
 4153 operand immI()
 4154 %{
 4155   match(ConI);
 4156 
 4157   op_cost(0);
 4158   format %{ %}
 4159   interface(CONST_INTER);
 4160 %}
 4161 
 4162 // 32 bit zero
 4163 operand immI0()
 4164 %{
 4165   predicate(n->get_int() == 0);
 4166   match(ConI);
 4167 
 4168   op_cost(0);
 4169   format %{ %}
 4170   interface(CONST_INTER);
 4171 %}
 4172 
 4173 // 32 bit unit increment
 4174 operand immI_1()
 4175 %{
 4176   predicate(n->get_int() == 1);
 4177   match(ConI);
 4178 
 4179   op_cost(0);
 4180   format %{ %}
 4181   interface(CONST_INTER);
 4182 %}
 4183 
 4184 // 32 bit unit decrement
 4185 operand immI_M1()
 4186 %{
 4187   predicate(n->get_int() == -1);
 4188   match(ConI);
 4189 
 4190   op_cost(0);
 4191   format %{ %}
 4192   interface(CONST_INTER);
 4193 %}
 4194 
 4195 // Shift values for add/sub extension shift
 4196 operand immIExt()
 4197 %{
 4198   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4199   match(ConI);
 4200 
 4201   op_cost(0);
 4202   format %{ %}
 4203   interface(CONST_INTER);
 4204 %}
 4205 
 4206 operand immI_gt_1()
 4207 %{
 4208   predicate(n->get_int() > 1);
 4209   match(ConI);
 4210 
 4211   op_cost(0);
 4212   format %{ %}
 4213   interface(CONST_INTER);
 4214 %}
 4215 
 4216 operand immI_le_4()
 4217 %{
 4218   predicate(n->get_int() <= 4);
 4219   match(ConI);
 4220 
 4221   op_cost(0);
 4222   format %{ %}
 4223   interface(CONST_INTER);
 4224 %}
 4225 
 4226 operand immI_31()
 4227 %{
 4228   predicate(n->get_int() == 31);
 4229   match(ConI);
 4230 
 4231   op_cost(0);
 4232   format %{ %}
 4233   interface(CONST_INTER);
 4234 %}
 4235 
 4236 operand immI_2()
 4237 %{
 4238   predicate(n->get_int() == 2);
 4239   match(ConI);
 4240 
 4241   op_cost(0);
 4242   format %{ %}
 4243   interface(CONST_INTER);
 4244 %}
 4245 
 4246 operand immI_4()
 4247 %{
 4248   predicate(n->get_int() == 4);
 4249   match(ConI);
 4250 
 4251   op_cost(0);
 4252   format %{ %}
 4253   interface(CONST_INTER);
 4254 %}
 4255 
 4256 operand immI_8()
 4257 %{
 4258   predicate(n->get_int() == 8);
 4259   match(ConI);
 4260 
 4261   op_cost(0);
 4262   format %{ %}
 4263   interface(CONST_INTER);
 4264 %}
 4265 
 4266 operand immI_16()
 4267 %{
 4268   predicate(n->get_int() == 16);
 4269   match(ConI);
 4270 
 4271   op_cost(0);
 4272   format %{ %}
 4273   interface(CONST_INTER);
 4274 %}
 4275 
 4276 operand immI_24()
 4277 %{
 4278   predicate(n->get_int() == 24);
 4279   match(ConI);
 4280 
 4281   op_cost(0);
 4282   format %{ %}
 4283   interface(CONST_INTER);
 4284 %}
 4285 
 4286 operand immI_32()
 4287 %{
 4288   predicate(n->get_int() == 32);
 4289   match(ConI);
 4290 
 4291   op_cost(0);
 4292   format %{ %}
 4293   interface(CONST_INTER);
 4294 %}
 4295 
 4296 operand immI_48()
 4297 %{
 4298   predicate(n->get_int() == 48);
 4299   match(ConI);
 4300 
 4301   op_cost(0);
 4302   format %{ %}
 4303   interface(CONST_INTER);
 4304 %}
 4305 
 4306 operand immI_56()
 4307 %{
 4308   predicate(n->get_int() == 56);
 4309   match(ConI);
 4310 
 4311   op_cost(0);
 4312   format %{ %}
 4313   interface(CONST_INTER);
 4314 %}
 4315 
 4316 operand immI_63()
 4317 %{
 4318   predicate(n->get_int() == 63);
 4319   match(ConI);
 4320 
 4321   op_cost(0);
 4322   format %{ %}
 4323   interface(CONST_INTER);
 4324 %}
 4325 
 4326 operand immI_64()
 4327 %{
 4328   predicate(n->get_int() == 64);
 4329   match(ConI);
 4330 
 4331   op_cost(0);
 4332   format %{ %}
 4333   interface(CONST_INTER);
 4334 %}
 4335 
 4336 operand immI_255()
 4337 %{
 4338   predicate(n->get_int() == 255);
 4339   match(ConI);
 4340 
 4341   op_cost(0);
 4342   format %{ %}
 4343   interface(CONST_INTER);
 4344 %}
 4345 
 4346 operand immI_65535()
 4347 %{
 4348   predicate(n->get_int() == 65535);
 4349   match(ConI);
 4350 
 4351   op_cost(0);
 4352   format %{ %}
 4353   interface(CONST_INTER);
 4354 %}
 4355 
 4356 operand immL_255()
 4357 %{
 4358   predicate(n->get_long() == 255L);
 4359   match(ConL);
 4360 
 4361   op_cost(0);
 4362   format %{ %}
 4363   interface(CONST_INTER);
 4364 %}
 4365 
 4366 operand immL_65535()
 4367 %{
 4368   predicate(n->get_long() == 65535L);
 4369   match(ConL);
 4370 
 4371   op_cost(0);
 4372   format %{ %}
 4373   interface(CONST_INTER);
 4374 %}
 4375 
 4376 operand immL_4294967295()
 4377 %{
 4378   predicate(n->get_long() == 4294967295L);
 4379   match(ConL);
 4380 
 4381   op_cost(0);
 4382   format %{ %}
 4383   interface(CONST_INTER);
 4384 %}
 4385 
 4386 operand immL_bitmask()
 4387 %{
 4388   predicate((n->get_long() != 0)
 4389             && ((n->get_long() & 0xc000000000000000l) == 0)
 4390             && is_power_of_2(n->get_long() + 1));
 4391   match(ConL);
 4392 
 4393   op_cost(0);
 4394   format %{ %}
 4395   interface(CONST_INTER);
 4396 %}
 4397 
 4398 operand immI_bitmask()
 4399 %{
 4400   predicate((n->get_int() != 0)
 4401             && ((n->get_int() & 0xc0000000) == 0)
 4402             && is_power_of_2(n->get_int() + 1));
 4403   match(ConI);
 4404 
 4405   op_cost(0);
 4406   format %{ %}
 4407   interface(CONST_INTER);
 4408 %}
 4409 
 4410 operand immL_positive_bitmaskI()
 4411 %{
 4412   predicate((n->get_long() != 0)
 4413             && ((julong)n->get_long() < 0x80000000ULL)
 4414             && is_power_of_2(n->get_long() + 1));
 4415   match(ConL);
 4416 
 4417   op_cost(0);
 4418   format %{ %}
 4419   interface(CONST_INTER);
 4420 %}
 4421 
 4422 // Scale values for scaled offset addressing modes (up to long but not quad)
 4423 operand immIScale()
 4424 %{
 4425   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4426   match(ConI);
 4427 
 4428   op_cost(0);
 4429   format %{ %}
 4430   interface(CONST_INTER);
 4431 %}
 4432 
 4433 // 26 bit signed offset -- for pc-relative branches
 4434 operand immI26()
 4435 %{
 4436   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4437   match(ConI);
 4438 
 4439   op_cost(0);
 4440   format %{ %}
 4441   interface(CONST_INTER);
 4442 %}
 4443 
 4444 // 19 bit signed offset -- for pc-relative loads
 4445 operand immI19()
 4446 %{
 4447   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4448   match(ConI);
 4449 
 4450   op_cost(0);
 4451   format %{ %}
 4452   interface(CONST_INTER);
 4453 %}
 4454 
 4455 // 12 bit unsigned offset -- for base plus immediate loads
 4456 operand immIU12()
 4457 %{
 4458   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4459   match(ConI);
 4460 
 4461   op_cost(0);
 4462   format %{ %}
 4463   interface(CONST_INTER);
 4464 %}
 4465 
 4466 operand immLU12()
 4467 %{
 4468   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4469   match(ConL);
 4470 
 4471   op_cost(0);
 4472   format %{ %}
 4473   interface(CONST_INTER);
 4474 %}
 4475 
 4476 // Offset for scaled or unscaled immediate loads and stores
 4477 operand immIOffset()
 4478 %{
 4479   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4480   match(ConI);
 4481 
 4482   op_cost(0);
 4483   format %{ %}
 4484   interface(CONST_INTER);
 4485 %}
 4486 
 4487 operand immIOffset1()
 4488 %{
 4489   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4490   match(ConI);
 4491 
 4492   op_cost(0);
 4493   format %{ %}
 4494   interface(CONST_INTER);
 4495 %}
 4496 
 4497 operand immIOffset2()
 4498 %{
 4499   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4500   match(ConI);
 4501 
 4502   op_cost(0);
 4503   format %{ %}
 4504   interface(CONST_INTER);
 4505 %}
 4506 
 4507 operand immIOffset4()
 4508 %{
 4509   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4510   match(ConI);
 4511 
 4512   op_cost(0);
 4513   format %{ %}
 4514   interface(CONST_INTER);
 4515 %}
 4516 
 4517 operand immIOffset8()
 4518 %{
 4519   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4520   match(ConI);
 4521 
 4522   op_cost(0);
 4523   format %{ %}
 4524   interface(CONST_INTER);
 4525 %}
 4526 
 4527 operand immIOffset16()
 4528 %{
 4529   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4530   match(ConI);
 4531 
 4532   op_cost(0);
 4533   format %{ %}
 4534   interface(CONST_INTER);
 4535 %}
 4536 
 4537 operand immLoffset()
 4538 %{
 4539   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4540   match(ConL);
 4541 
 4542   op_cost(0);
 4543   format %{ %}
 4544   interface(CONST_INTER);
 4545 %}
 4546 
 4547 operand immLoffset1()
 4548 %{
 4549   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4550   match(ConL);
 4551 
 4552   op_cost(0);
 4553   format %{ %}
 4554   interface(CONST_INTER);
 4555 %}
 4556 
 4557 operand immLoffset2()
 4558 %{
 4559   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4560   match(ConL);
 4561 
 4562   op_cost(0);
 4563   format %{ %}
 4564   interface(CONST_INTER);
 4565 %}
 4566 
 4567 operand immLoffset4()
 4568 %{
 4569   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4570   match(ConL);
 4571 
 4572   op_cost(0);
 4573   format %{ %}
 4574   interface(CONST_INTER);
 4575 %}
 4576 
 4577 operand immLoffset8()
 4578 %{
 4579   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4580   match(ConL);
 4581 
 4582   op_cost(0);
 4583   format %{ %}
 4584   interface(CONST_INTER);
 4585 %}
 4586 
 4587 operand immLoffset16()
 4588 %{
 4589   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4590   match(ConL);
 4591 
 4592   op_cost(0);
 4593   format %{ %}
 4594   interface(CONST_INTER);
 4595 %}
 4596 
 4597 // 8 bit signed value.
 4598 operand immI8()
 4599 %{
 4600   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4601   match(ConI);
 4602 
 4603   op_cost(0);
 4604   format %{ %}
 4605   interface(CONST_INTER);
 4606 %}
 4607 
 4608 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4609 operand immI8_shift8()
 4610 %{
 4611   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4612             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4613   match(ConI);
 4614 
 4615   op_cost(0);
 4616   format %{ %}
 4617   interface(CONST_INTER);
 4618 %}
 4619 
 4620 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4621 operand immL8_shift8()
 4622 %{
 4623   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4624             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4625   match(ConL);
 4626 
 4627   op_cost(0);
 4628   format %{ %}
 4629   interface(CONST_INTER);
 4630 %}
 4631 
 4632 // 32 bit integer valid for add sub immediate
 4633 operand immIAddSub()
 4634 %{
 4635   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4636   match(ConI);
 4637   op_cost(0);
 4638   format %{ %}
 4639   interface(CONST_INTER);
 4640 %}
 4641 
 4642 // 32 bit unsigned integer valid for logical immediate
 4643 // TODO -- check this is right when e.g the mask is 0x80000000
 4644 operand immILog()
 4645 %{
 4646   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4647   match(ConI);
 4648 
 4649   op_cost(0);
 4650   format %{ %}
 4651   interface(CONST_INTER);
 4652 %}
 4653 
 4654 // Integer operands 64 bit
 4655 // 64 bit immediate
 4656 operand immL()
 4657 %{
 4658   match(ConL);
 4659 
 4660   op_cost(0);
 4661   format %{ %}
 4662   interface(CONST_INTER);
 4663 %}
 4664 
 4665 // 64 bit zero
 4666 operand immL0()
 4667 %{
 4668   predicate(n->get_long() == 0);
 4669   match(ConL);
 4670 
 4671   op_cost(0);
 4672   format %{ %}
 4673   interface(CONST_INTER);
 4674 %}
 4675 
 4676 // 64 bit unit increment
 4677 operand immL_1()
 4678 %{
 4679   predicate(n->get_long() == 1);
 4680   match(ConL);
 4681 
 4682   op_cost(0);
 4683   format %{ %}
 4684   interface(CONST_INTER);
 4685 %}
 4686 
 4687 // 64 bit unit decrement
 4688 operand immL_M1()
 4689 %{
 4690   predicate(n->get_long() == -1);
 4691   match(ConL);
 4692 
 4693   op_cost(0);
 4694   format %{ %}
 4695   interface(CONST_INTER);
 4696 %}
 4697 
 4698 // 32 bit offset of pc in thread anchor
 4699 
 4700 operand immL_pc_off()
 4701 %{
 4702   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4703                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4704   match(ConL);
 4705 
 4706   op_cost(0);
 4707   format %{ %}
 4708   interface(CONST_INTER);
 4709 %}
 4710 
 4711 // 64 bit integer valid for add sub immediate
 4712 operand immLAddSub()
 4713 %{
 4714   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4715   match(ConL);
 4716   op_cost(0);
 4717   format %{ %}
 4718   interface(CONST_INTER);
 4719 %}
 4720 
 4721 // 64 bit integer valid for logical immediate
 4722 operand immLLog()
 4723 %{
 4724   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4725   match(ConL);
 4726   op_cost(0);
 4727   format %{ %}
 4728   interface(CONST_INTER);
 4729 %}
 4730 
 4731 // Long Immediate: low 32-bit mask
 4732 operand immL_32bits()
 4733 %{
 4734   predicate(n->get_long() == 0xFFFFFFFFL);
 4735   match(ConL);
 4736   op_cost(0);
 4737   format %{ %}
 4738   interface(CONST_INTER);
 4739 %}
 4740 
 4741 // Pointer operands
 4742 // Pointer Immediate
 4743 operand immP()
 4744 %{
 4745   match(ConP);
 4746 
 4747   op_cost(0);
 4748   format %{ %}
 4749   interface(CONST_INTER);
 4750 %}
 4751 
 4752 // NULL Pointer Immediate
 4753 operand immP0()
 4754 %{
 4755   predicate(n->get_ptr() == 0);
 4756   match(ConP);
 4757 
 4758   op_cost(0);
 4759   format %{ %}
 4760   interface(CONST_INTER);
 4761 %}
 4762 
 4763 // Pointer Immediate One
 4764 // this is used in object initialization (initial object header)
 4765 operand immP_1()
 4766 %{
 4767   predicate(n->get_ptr() == 1);
 4768   match(ConP);
 4769 
 4770   op_cost(0);
 4771   format %{ %}
 4772   interface(CONST_INTER);
 4773 %}
 4774 
 4775 // Card Table Byte Map Base
 4776 operand immByteMapBase()
 4777 %{
 4778   // Get base of card map
 4779   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4780             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4781   match(ConP);
 4782 
 4783   op_cost(0);
 4784   format %{ %}
 4785   interface(CONST_INTER);
 4786 %}
 4787 
 4788 // Pointer Immediate Minus One
 4789 // this is used when we want to write the current PC to the thread anchor
 4790 operand immP_M1()
 4791 %{
 4792   predicate(n->get_ptr() == -1);
 4793   match(ConP);
 4794 
 4795   op_cost(0);
 4796   format %{ %}
 4797   interface(CONST_INTER);
 4798 %}
 4799 
 4800 // Pointer Immediate Minus Two
 4801 // this is used when we want to write the current PC to the thread anchor
 4802 operand immP_M2()
 4803 %{
 4804   predicate(n->get_ptr() == -2);
 4805   match(ConP);
 4806 
 4807   op_cost(0);
 4808   format %{ %}
 4809   interface(CONST_INTER);
 4810 %}
 4811 
 4812 // Float and Double operands
 4813 // Double Immediate
 4814 operand immD()
 4815 %{
 4816   match(ConD);
 4817   op_cost(0);
 4818   format %{ %}
 4819   interface(CONST_INTER);
 4820 %}
 4821 
 4822 // Double Immediate: +0.0d
 4823 operand immD0()
 4824 %{
 4825   predicate(jlong_cast(n->getd()) == 0);
 4826   match(ConD);
 4827 
 4828   op_cost(0);
 4829   format %{ %}
 4830   interface(CONST_INTER);
 4831 %}
 4832 
 4833 // constant 'double +0.0'.
 4834 operand immDPacked()
 4835 %{
 4836   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4837   match(ConD);
 4838   op_cost(0);
 4839   format %{ %}
 4840   interface(CONST_INTER);
 4841 %}
 4842 
 4843 // Float Immediate
 4844 operand immF()
 4845 %{
 4846   match(ConF);
 4847   op_cost(0);
 4848   format %{ %}
 4849   interface(CONST_INTER);
 4850 %}
 4851 
 4852 // Float Immediate: +0.0f.
 4853 operand immF0()
 4854 %{
 4855   predicate(jint_cast(n->getf()) == 0);
 4856   match(ConF);
 4857 
 4858   op_cost(0);
 4859   format %{ %}
 4860   interface(CONST_INTER);
 4861 %}
 4862 
 4863 //
 4864 operand immFPacked()
 4865 %{
 4866   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4867   match(ConF);
 4868   op_cost(0);
 4869   format %{ %}
 4870   interface(CONST_INTER);
 4871 %}
 4872 
 4873 // Narrow pointer operands
 4874 // Narrow Pointer Immediate
 4875 operand immN()
 4876 %{
 4877   match(ConN);
 4878 
 4879   op_cost(0);
 4880   format %{ %}
 4881   interface(CONST_INTER);
 4882 %}
 4883 
 4884 // Narrow NULL Pointer Immediate
 4885 operand immN0()
 4886 %{
 4887   predicate(n->get_narrowcon() == 0);
 4888   match(ConN);
 4889 
 4890   op_cost(0);
 4891   format %{ %}
 4892   interface(CONST_INTER);
 4893 %}
 4894 
 4895 operand immNKlass()
 4896 %{
 4897   match(ConNKlass);
 4898 
 4899   op_cost(0);
 4900   format %{ %}
 4901   interface(CONST_INTER);
 4902 %}
 4903 
 4904 // Integer 32 bit Register Operands
 4905 // Integer 32 bitRegister (excludes SP)
 4906 operand iRegI()
 4907 %{
 4908   constraint(ALLOC_IN_RC(any_reg32));
 4909   match(RegI);
 4910   match(iRegINoSp);
 4911   op_cost(0);
 4912   format %{ %}
 4913   interface(REG_INTER);
 4914 %}
 4915 
 4916 // Integer 32 bit Register not Special
 4917 operand iRegINoSp()
 4918 %{
 4919   constraint(ALLOC_IN_RC(no_special_reg32));
 4920   match(RegI);
 4921   op_cost(0);
 4922   format %{ %}
 4923   interface(REG_INTER);
 4924 %}
 4925 
 4926 // Integer 64 bit Register Operands
 4927 // Integer 64 bit Register (includes SP)
 4928 operand iRegL()
 4929 %{
 4930   constraint(ALLOC_IN_RC(any_reg));
 4931   match(RegL);
 4932   match(iRegLNoSp);
 4933   op_cost(0);
 4934   format %{ %}
 4935   interface(REG_INTER);
 4936 %}
 4937 
 4938 // Integer 64 bit Register not Special
 4939 operand iRegLNoSp()
 4940 %{
 4941   constraint(ALLOC_IN_RC(no_special_reg));
 4942   match(RegL);
 4943   match(iRegL_R0);
 4944   format %{ %}
 4945   interface(REG_INTER);
 4946 %}
 4947 
 4948 // Pointer Register Operands
 4949 // Pointer Register
 4950 operand iRegP()
 4951 %{
 4952   constraint(ALLOC_IN_RC(ptr_reg));
 4953   match(RegP);
 4954   match(iRegPNoSp);
 4955   match(iRegP_R0);
 4956   //match(iRegP_R2);
 4957   //match(iRegP_R4);
 4958   //match(iRegP_R5);
 4959   match(thread_RegP);
 4960   op_cost(0);
 4961   format %{ %}
 4962   interface(REG_INTER);
 4963 %}
 4964 
 4965 // Pointer 64 bit Register not Special
 4966 operand iRegPNoSp()
 4967 %{
 4968   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4969   match(RegP);
 4970   // match(iRegP);
 4971   // match(iRegP_R0);
 4972   // match(iRegP_R2);
 4973   // match(iRegP_R4);
 4974   // match(iRegP_R5);
 4975   // match(thread_RegP);
 4976   op_cost(0);
 4977   format %{ %}
 4978   interface(REG_INTER);
 4979 %}
 4980 
 4981 // Pointer 64 bit Register R0 only
 4982 operand iRegP_R0()
 4983 %{
 4984   constraint(ALLOC_IN_RC(r0_reg));
 4985   match(RegP);
 4986   // match(iRegP);
 4987   match(iRegPNoSp);
 4988   op_cost(0);
 4989   format %{ %}
 4990   interface(REG_INTER);
 4991 %}
 4992 
 4993 // Pointer 64 bit Register R1 only
 4994 operand iRegP_R1()
 4995 %{
 4996   constraint(ALLOC_IN_RC(r1_reg));
 4997   match(RegP);
 4998   // match(iRegP);
 4999   match(iRegPNoSp);
 5000   op_cost(0);
 5001   format %{ %}
 5002   interface(REG_INTER);
 5003 %}
 5004 
 5005 // Pointer 64 bit Register R2 only
 5006 operand iRegP_R2()
 5007 %{
 5008   constraint(ALLOC_IN_RC(r2_reg));
 5009   match(RegP);
 5010   // match(iRegP);
 5011   match(iRegPNoSp);
 5012   op_cost(0);
 5013   format %{ %}
 5014   interface(REG_INTER);
 5015 %}
 5016 
 5017 // Pointer 64 bit Register R3 only
 5018 operand iRegP_R3()
 5019 %{
 5020   constraint(ALLOC_IN_RC(r3_reg));
 5021   match(RegP);
 5022   // match(iRegP);
 5023   match(iRegPNoSp);
 5024   op_cost(0);
 5025   format %{ %}
 5026   interface(REG_INTER);
 5027 %}
 5028 
 5029 // Pointer 64 bit Register R4 only
 5030 operand iRegP_R4()
 5031 %{
 5032   constraint(ALLOC_IN_RC(r4_reg));
 5033   match(RegP);
 5034   // match(iRegP);
 5035   match(iRegPNoSp);
 5036   op_cost(0);
 5037   format %{ %}
 5038   interface(REG_INTER);
 5039 %}
 5040 
 5041 // Pointer 64 bit Register R5 only
 5042 operand iRegP_R5()
 5043 %{
 5044   constraint(ALLOC_IN_RC(r5_reg));
 5045   match(RegP);
 5046   // match(iRegP);
 5047   match(iRegPNoSp);
 5048   op_cost(0);
 5049   format %{ %}
 5050   interface(REG_INTER);
 5051 %}
 5052 
 5053 // Pointer 64 bit Register R10 only
 5054 operand iRegP_R10()
 5055 %{
 5056   constraint(ALLOC_IN_RC(r10_reg));
 5057   match(RegP);
 5058   // match(iRegP);
 5059   match(iRegPNoSp);
 5060   op_cost(0);
 5061   format %{ %}
 5062   interface(REG_INTER);
 5063 %}
 5064 
 5065 // Long 64 bit Register R0 only
 5066 operand iRegL_R0()
 5067 %{
 5068   constraint(ALLOC_IN_RC(r0_reg));
 5069   match(RegL);
 5070   match(iRegLNoSp);
 5071   op_cost(0);
 5072   format %{ %}
 5073   interface(REG_INTER);
 5074 %}
 5075 
 5076 // Long 64 bit Register R2 only
 5077 operand iRegL_R2()
 5078 %{
 5079   constraint(ALLOC_IN_RC(r2_reg));
 5080   match(RegL);
 5081   match(iRegLNoSp);
 5082   op_cost(0);
 5083   format %{ %}
 5084   interface(REG_INTER);
 5085 %}
 5086 
 5087 // Long 64 bit Register R3 only
 5088 operand iRegL_R3()
 5089 %{
 5090   constraint(ALLOC_IN_RC(r3_reg));
 5091   match(RegL);
 5092   match(iRegLNoSp);
 5093   op_cost(0);
 5094   format %{ %}
 5095   interface(REG_INTER);
 5096 %}
 5097 
 5098 // Long 64 bit Register R11 only
 5099 operand iRegL_R11()
 5100 %{
 5101   constraint(ALLOC_IN_RC(r11_reg));
 5102   match(RegL);
 5103   match(iRegLNoSp);
 5104   op_cost(0);
 5105   format %{ %}
 5106   interface(REG_INTER);
 5107 %}
 5108 
 5109 // Pointer 64 bit Register FP only
 5110 operand iRegP_FP()
 5111 %{
 5112   constraint(ALLOC_IN_RC(fp_reg));
 5113   match(RegP);
 5114   // match(iRegP);
 5115   op_cost(0);
 5116   format %{ %}
 5117   interface(REG_INTER);
 5118 %}
 5119 
 5120 // Register R0 only
 5121 operand iRegI_R0()
 5122 %{
 5123   constraint(ALLOC_IN_RC(int_r0_reg));
 5124   match(RegI);
 5125   match(iRegINoSp);
 5126   op_cost(0);
 5127   format %{ %}
 5128   interface(REG_INTER);
 5129 %}
 5130 
 5131 // Register R2 only
 5132 operand iRegI_R2()
 5133 %{
 5134   constraint(ALLOC_IN_RC(int_r2_reg));
 5135   match(RegI);
 5136   match(iRegINoSp);
 5137   op_cost(0);
 5138   format %{ %}
 5139   interface(REG_INTER);
 5140 %}
 5141 
 5142 // Register R3 only
 5143 operand iRegI_R3()
 5144 %{
 5145   constraint(ALLOC_IN_RC(int_r3_reg));
 5146   match(RegI);
 5147   match(iRegINoSp);
 5148   op_cost(0);
 5149   format %{ %}
 5150   interface(REG_INTER);
 5151 %}
 5152 
 5153 
 5154 // Register R4 only
 5155 operand iRegI_R4()
 5156 %{
 5157   constraint(ALLOC_IN_RC(int_r4_reg));
 5158   match(RegI);
 5159   match(iRegINoSp);
 5160   op_cost(0);
 5161   format %{ %}
 5162   interface(REG_INTER);
 5163 %}
 5164 
 5165 
 5166 // Pointer Register Operands
 5167 // Narrow Pointer Register
 5168 operand iRegN()
 5169 %{
 5170   constraint(ALLOC_IN_RC(any_reg32));
 5171   match(RegN);
 5172   match(iRegNNoSp);
 5173   op_cost(0);
 5174   format %{ %}
 5175   interface(REG_INTER);
 5176 %}
 5177 
 5178 operand iRegN_R0()
 5179 %{
 5180   constraint(ALLOC_IN_RC(r0_reg));
 5181   match(iRegN);
 5182   op_cost(0);
 5183   format %{ %}
 5184   interface(REG_INTER);
 5185 %}
 5186 
 5187 operand iRegN_R2()
 5188 %{
 5189   constraint(ALLOC_IN_RC(r2_reg));
 5190   match(iRegN);
 5191   op_cost(0);
 5192   format %{ %}
 5193   interface(REG_INTER);
 5194 %}
 5195 
 5196 operand iRegN_R3()
 5197 %{
 5198   constraint(ALLOC_IN_RC(r3_reg));
 5199   match(iRegN);
 5200   op_cost(0);
 5201   format %{ %}
 5202   interface(REG_INTER);
 5203 %}
 5204 
 5205 // Integer 64 bit Register not Special
 5206 operand iRegNNoSp()
 5207 %{
 5208   constraint(ALLOC_IN_RC(no_special_reg32));
 5209   match(RegN);
 5210   op_cost(0);
 5211   format %{ %}
 5212   interface(REG_INTER);
 5213 %}
 5214 
 5215 // heap base register -- used for encoding immN0
 5216 
 5217 operand iRegIHeapbase()
 5218 %{
 5219   constraint(ALLOC_IN_RC(heapbase_reg));
 5220   match(RegI);
 5221   op_cost(0);
 5222   format %{ %}
 5223   interface(REG_INTER);
 5224 %}
 5225 
 5226 // Float Register
 5227 // Float register operands
 5228 operand vRegF()
 5229 %{
 5230   constraint(ALLOC_IN_RC(float_reg));
 5231   match(RegF);
 5232 
 5233   op_cost(0);
 5234   format %{ %}
 5235   interface(REG_INTER);
 5236 %}
 5237 
 5238 // Double Register
 5239 // Double register operands
 5240 operand vRegD()
 5241 %{
 5242   constraint(ALLOC_IN_RC(double_reg));
 5243   match(RegD);
 5244 
 5245   op_cost(0);
 5246   format %{ %}
 5247   interface(REG_INTER);
 5248 %}
 5249 
 5250 // Generic vector class. This will be used for
 5251 // all vector operands, including NEON and SVE,
 5252 // but currently only used for SVE VecA.
 5253 operand vReg()
 5254 %{
 5255   constraint(ALLOC_IN_RC(vectora_reg));
 5256   match(VecA);
 5257   op_cost(0);
 5258   format %{ %}
 5259   interface(REG_INTER);
 5260 %}
 5261 
 5262 operand vecD()
 5263 %{
 5264   constraint(ALLOC_IN_RC(vectord_reg));
 5265   match(VecD);
 5266 
 5267   op_cost(0);
 5268   format %{ %}
 5269   interface(REG_INTER);
 5270 %}
 5271 
 5272 operand vecX()
 5273 %{
 5274   constraint(ALLOC_IN_RC(vectorx_reg));
 5275   match(VecX);
 5276 
 5277   op_cost(0);
 5278   format %{ %}
 5279   interface(REG_INTER);
 5280 %}
 5281 
 5282 operand vRegD_V0()
 5283 %{
 5284   constraint(ALLOC_IN_RC(v0_reg));
 5285   match(RegD);
 5286   op_cost(0);
 5287   format %{ %}
 5288   interface(REG_INTER);
 5289 %}
 5290 
 5291 operand vRegD_V1()
 5292 %{
 5293   constraint(ALLOC_IN_RC(v1_reg));
 5294   match(RegD);
 5295   op_cost(0);
 5296   format %{ %}
 5297   interface(REG_INTER);
 5298 %}
 5299 
 5300 operand vRegD_V2()
 5301 %{
 5302   constraint(ALLOC_IN_RC(v2_reg));
 5303   match(RegD);
 5304   op_cost(0);
 5305   format %{ %}
 5306   interface(REG_INTER);
 5307 %}
 5308 
 5309 operand vRegD_V3()
 5310 %{
 5311   constraint(ALLOC_IN_RC(v3_reg));
 5312   match(RegD);
 5313   op_cost(0);
 5314   format %{ %}
 5315   interface(REG_INTER);
 5316 %}
 5317 
 5318 operand vRegD_V4()
 5319 %{
 5320   constraint(ALLOC_IN_RC(v4_reg));
 5321   match(RegD);
 5322   op_cost(0);
 5323   format %{ %}
 5324   interface(REG_INTER);
 5325 %}
 5326 
 5327 operand vRegD_V5()
 5328 %{
 5329   constraint(ALLOC_IN_RC(v5_reg));
 5330   match(RegD);
 5331   op_cost(0);
 5332   format %{ %}
 5333   interface(REG_INTER);
 5334 %}
 5335 
 5336 operand vRegD_V6()
 5337 %{
 5338   constraint(ALLOC_IN_RC(v6_reg));
 5339   match(RegD);
 5340   op_cost(0);
 5341   format %{ %}
 5342   interface(REG_INTER);
 5343 %}
 5344 
 5345 operand vRegD_V7()
 5346 %{
 5347   constraint(ALLOC_IN_RC(v7_reg));
 5348   match(RegD);
 5349   op_cost(0);
 5350   format %{ %}
 5351   interface(REG_INTER);
 5352 %}
 5353 
 5354 operand vRegD_V8()
 5355 %{
 5356   constraint(ALLOC_IN_RC(v8_reg));
 5357   match(RegD);
 5358   op_cost(0);
 5359   format %{ %}
 5360   interface(REG_INTER);
 5361 %}
 5362 
 5363 operand vRegD_V9()
 5364 %{
 5365   constraint(ALLOC_IN_RC(v9_reg));
 5366   match(RegD);
 5367   op_cost(0);
 5368   format %{ %}
 5369   interface(REG_INTER);
 5370 %}
 5371 
 5372 operand vRegD_V10()
 5373 %{
 5374   constraint(ALLOC_IN_RC(v10_reg));
 5375   match(RegD);
 5376   op_cost(0);
 5377   format %{ %}
 5378   interface(REG_INTER);
 5379 %}
 5380 
 5381 operand vRegD_V11()
 5382 %{
 5383   constraint(ALLOC_IN_RC(v11_reg));
 5384   match(RegD);
 5385   op_cost(0);
 5386   format %{ %}
 5387   interface(REG_INTER);
 5388 %}
 5389 
 5390 operand vRegD_V12()
 5391 %{
 5392   constraint(ALLOC_IN_RC(v12_reg));
 5393   match(RegD);
 5394   op_cost(0);
 5395   format %{ %}
 5396   interface(REG_INTER);
 5397 %}
 5398 
 5399 operand vRegD_V13()
 5400 %{
 5401   constraint(ALLOC_IN_RC(v13_reg));
 5402   match(RegD);
 5403   op_cost(0);
 5404   format %{ %}
 5405   interface(REG_INTER);
 5406 %}
 5407 
 5408 operand vRegD_V14()
 5409 %{
 5410   constraint(ALLOC_IN_RC(v14_reg));
 5411   match(RegD);
 5412   op_cost(0);
 5413   format %{ %}
 5414   interface(REG_INTER);
 5415 %}
 5416 
 5417 operand vRegD_V15()
 5418 %{
 5419   constraint(ALLOC_IN_RC(v15_reg));
 5420   match(RegD);
 5421   op_cost(0);
 5422   format %{ %}
 5423   interface(REG_INTER);
 5424 %}
 5425 
 5426 operand vRegD_V16()
 5427 %{
 5428   constraint(ALLOC_IN_RC(v16_reg));
 5429   match(RegD);
 5430   op_cost(0);
 5431   format %{ %}
 5432   interface(REG_INTER);
 5433 %}
 5434 
 5435 operand vRegD_V17()
 5436 %{
 5437   constraint(ALLOC_IN_RC(v17_reg));
 5438   match(RegD);
 5439   op_cost(0);
 5440   format %{ %}
 5441   interface(REG_INTER);
 5442 %}
 5443 
 5444 operand vRegD_V18()
 5445 %{
 5446   constraint(ALLOC_IN_RC(v18_reg));
 5447   match(RegD);
 5448   op_cost(0);
 5449   format %{ %}
 5450   interface(REG_INTER);
 5451 %}
 5452 
 5453 operand vRegD_V19()
 5454 %{
 5455   constraint(ALLOC_IN_RC(v19_reg));
 5456   match(RegD);
 5457   op_cost(0);
 5458   format %{ %}
 5459   interface(REG_INTER);
 5460 %}
 5461 
 5462 operand vRegD_V20()
 5463 %{
 5464   constraint(ALLOC_IN_RC(v20_reg));
 5465   match(RegD);
 5466   op_cost(0);
 5467   format %{ %}
 5468   interface(REG_INTER);
 5469 %}
 5470 
 5471 operand vRegD_V21()
 5472 %{
 5473   constraint(ALLOC_IN_RC(v21_reg));
 5474   match(RegD);
 5475   op_cost(0);
 5476   format %{ %}
 5477   interface(REG_INTER);
 5478 %}
 5479 
 5480 operand vRegD_V22()
 5481 %{
 5482   constraint(ALLOC_IN_RC(v22_reg));
 5483   match(RegD);
 5484   op_cost(0);
 5485   format %{ %}
 5486   interface(REG_INTER);
 5487 %}
 5488 
 5489 operand vRegD_V23()
 5490 %{
 5491   constraint(ALLOC_IN_RC(v23_reg));
 5492   match(RegD);
 5493   op_cost(0);
 5494   format %{ %}
 5495   interface(REG_INTER);
 5496 %}
 5497 
 5498 operand vRegD_V24()
 5499 %{
 5500   constraint(ALLOC_IN_RC(v24_reg));
 5501   match(RegD);
 5502   op_cost(0);
 5503   format %{ %}
 5504   interface(REG_INTER);
 5505 %}
 5506 
 5507 operand vRegD_V25()
 5508 %{
 5509   constraint(ALLOC_IN_RC(v25_reg));
 5510   match(RegD);
 5511   op_cost(0);
 5512   format %{ %}
 5513   interface(REG_INTER);
 5514 %}
 5515 
 5516 operand vRegD_V26()
 5517 %{
 5518   constraint(ALLOC_IN_RC(v26_reg));
 5519   match(RegD);
 5520   op_cost(0);
 5521   format %{ %}
 5522   interface(REG_INTER);
 5523 %}
 5524 
 5525 operand vRegD_V27()
 5526 %{
 5527   constraint(ALLOC_IN_RC(v27_reg));
 5528   match(RegD);
 5529   op_cost(0);
 5530   format %{ %}
 5531   interface(REG_INTER);
 5532 %}
 5533 
 5534 operand vRegD_V28()
 5535 %{
 5536   constraint(ALLOC_IN_RC(v28_reg));
 5537   match(RegD);
 5538   op_cost(0);
 5539   format %{ %}
 5540   interface(REG_INTER);
 5541 %}
 5542 
 5543 operand vRegD_V29()
 5544 %{
 5545   constraint(ALLOC_IN_RC(v29_reg));
 5546   match(RegD);
 5547   op_cost(0);
 5548   format %{ %}
 5549   interface(REG_INTER);
 5550 %}
 5551 
 5552 operand vRegD_V30()
 5553 %{
 5554   constraint(ALLOC_IN_RC(v30_reg));
 5555   match(RegD);
 5556   op_cost(0);
 5557   format %{ %}
 5558   interface(REG_INTER);
 5559 %}
 5560 
 5561 operand vRegD_V31()
 5562 %{
 5563   constraint(ALLOC_IN_RC(v31_reg));
 5564   match(RegD);
 5565   op_cost(0);
 5566   format %{ %}
 5567   interface(REG_INTER);
 5568 %}
 5569 
 5570 operand pReg()
 5571 %{
 5572   constraint(ALLOC_IN_RC(pr_reg));
 5573   match(RegVectMask);
 5574   match(pRegGov);
 5575   op_cost(0);
 5576   format %{ %}
 5577   interface(REG_INTER);
 5578 %}
 5579 
 5580 operand pRegGov()
 5581 %{
 5582   constraint(ALLOC_IN_RC(gov_pr));
 5583   match(RegVectMask);
 5584   op_cost(0);
 5585   format %{ %}
 5586   interface(REG_INTER);
 5587 %}
 5588 
 5589 // Flags register, used as output of signed compare instructions
 5590 
 5591 // note that on AArch64 we also use this register as the output for
 5592 // for floating point compare instructions (CmpF CmpD). this ensures
 5593 // that ordered inequality tests use GT, GE, LT or LE none of which
 5594 // pass through cases where the result is unordered i.e. one or both
 5595 // inputs to the compare is a NaN. this means that the ideal code can
 5596 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5597 // (where the comparison should always fail). EQ and NE tests are
 5598 // always generated in ideal code so that unordered folds into the NE
 5599 // case, matching the behaviour of AArch64 NE.
 5600 //
 5601 // This differs from x86 where the outputs of FP compares use a
 5602 // special FP flags registers and where compares based on this
 5603 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5604 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5605 // to explicitly handle the unordered case in branches. x86 also has
 5606 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5607 
 5608 operand rFlagsReg()
 5609 %{
 5610   constraint(ALLOC_IN_RC(int_flags));
 5611   match(RegFlags);
 5612 
 5613   op_cost(0);
 5614   format %{ "RFLAGS" %}
 5615   interface(REG_INTER);
 5616 %}
 5617 
 5618 // Flags register, used as output of unsigned compare instructions
 5619 operand rFlagsRegU()
 5620 %{
 5621   constraint(ALLOC_IN_RC(int_flags));
 5622   match(RegFlags);
 5623 
 5624   op_cost(0);
 5625   format %{ "RFLAGSU" %}
 5626   interface(REG_INTER);
 5627 %}
 5628 
 5629 // Special Registers
 5630 
 5631 // Method Register
 5632 operand inline_cache_RegP(iRegP reg)
 5633 %{
 5634   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5635   match(reg);
 5636   match(iRegPNoSp);
 5637   op_cost(0);
 5638   format %{ %}
 5639   interface(REG_INTER);
 5640 %}
 5641 
 5642 // Thread Register
 5643 operand thread_RegP(iRegP reg)
 5644 %{
 5645   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5646   match(reg);
 5647   op_cost(0);
 5648   format %{ %}
 5649   interface(REG_INTER);
 5650 %}
 5651 
 5652 operand lr_RegP(iRegP reg)
 5653 %{
 5654   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5655   match(reg);
 5656   op_cost(0);
 5657   format %{ %}
 5658   interface(REG_INTER);
 5659 %}
 5660 
 5661 //----------Memory Operands----------------------------------------------------
 5662 
 5663 operand indirect(iRegP reg)
 5664 %{
 5665   constraint(ALLOC_IN_RC(ptr_reg));
 5666   match(reg);
 5667   op_cost(0);
 5668   format %{ "[$reg]" %}
 5669   interface(MEMORY_INTER) %{
 5670     base($reg);
 5671     index(0xffffffff);
 5672     scale(0x0);
 5673     disp(0x0);
 5674   %}
 5675 %}
 5676 
 5677 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5678 %{
 5679   constraint(ALLOC_IN_RC(ptr_reg));
 5680   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5681   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5682   op_cost(0);
 5683   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5684   interface(MEMORY_INTER) %{
 5685     base($reg);
 5686     index($ireg);
 5687     scale($scale);
 5688     disp(0x0);
 5689   %}
 5690 %}
 5691 
 5692 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5693 %{
 5694   constraint(ALLOC_IN_RC(ptr_reg));
 5695   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5696   match(AddP reg (LShiftL lreg scale));
 5697   op_cost(0);
 5698   format %{ "$reg, $lreg lsl($scale)" %}
 5699   interface(MEMORY_INTER) %{
 5700     base($reg);
 5701     index($lreg);
 5702     scale($scale);
 5703     disp(0x0);
 5704   %}
 5705 %}
 5706 
 5707 operand indIndexI2L(iRegP reg, iRegI ireg)
 5708 %{
 5709   constraint(ALLOC_IN_RC(ptr_reg));
 5710   match(AddP reg (ConvI2L ireg));
 5711   op_cost(0);
 5712   format %{ "$reg, $ireg, 0, I2L" %}
 5713   interface(MEMORY_INTER) %{
 5714     base($reg);
 5715     index($ireg);
 5716     scale(0x0);
 5717     disp(0x0);
 5718   %}
 5719 %}
 5720 
 5721 operand indIndex(iRegP reg, iRegL lreg)
 5722 %{
 5723   constraint(ALLOC_IN_RC(ptr_reg));
 5724   match(AddP reg lreg);
 5725   op_cost(0);
 5726   format %{ "$reg, $lreg" %}
 5727   interface(MEMORY_INTER) %{
 5728     base($reg);
 5729     index($lreg);
 5730     scale(0x0);
 5731     disp(0x0);
 5732   %}
 5733 %}
 5734 
 5735 operand indOffI(iRegP reg, immIOffset off)
 5736 %{
 5737   constraint(ALLOC_IN_RC(ptr_reg));
 5738   match(AddP reg off);
 5739   op_cost(0);
 5740   format %{ "[$reg, $off]" %}
 5741   interface(MEMORY_INTER) %{
 5742     base($reg);
 5743     index(0xffffffff);
 5744     scale(0x0);
 5745     disp($off);
 5746   %}
 5747 %}
 5748 
 5749 operand indOffI1(iRegP reg, immIOffset1 off)
 5750 %{
 5751   constraint(ALLOC_IN_RC(ptr_reg));
 5752   match(AddP reg off);
 5753   op_cost(0);
 5754   format %{ "[$reg, $off]" %}
 5755   interface(MEMORY_INTER) %{
 5756     base($reg);
 5757     index(0xffffffff);
 5758     scale(0x0);
 5759     disp($off);
 5760   %}
 5761 %}
 5762 
 5763 operand indOffI2(iRegP reg, immIOffset2 off)
 5764 %{
 5765   constraint(ALLOC_IN_RC(ptr_reg));
 5766   match(AddP reg off);
 5767   op_cost(0);
 5768   format %{ "[$reg, $off]" %}
 5769   interface(MEMORY_INTER) %{
 5770     base($reg);
 5771     index(0xffffffff);
 5772     scale(0x0);
 5773     disp($off);
 5774   %}
 5775 %}
 5776 
 5777 operand indOffI4(iRegP reg, immIOffset4 off)
 5778 %{
 5779   constraint(ALLOC_IN_RC(ptr_reg));
 5780   match(AddP reg off);
 5781   op_cost(0);
 5782   format %{ "[$reg, $off]" %}
 5783   interface(MEMORY_INTER) %{
 5784     base($reg);
 5785     index(0xffffffff);
 5786     scale(0x0);
 5787     disp($off);
 5788   %}
 5789 %}
 5790 
 5791 operand indOffI8(iRegP reg, immIOffset8 off)
 5792 %{
 5793   constraint(ALLOC_IN_RC(ptr_reg));
 5794   match(AddP reg off);
 5795   op_cost(0);
 5796   format %{ "[$reg, $off]" %}
 5797   interface(MEMORY_INTER) %{
 5798     base($reg);
 5799     index(0xffffffff);
 5800     scale(0x0);
 5801     disp($off);
 5802   %}
 5803 %}
 5804 
 5805 operand indOffI16(iRegP reg, immIOffset16 off)
 5806 %{
 5807   constraint(ALLOC_IN_RC(ptr_reg));
 5808   match(AddP reg off);
 5809   op_cost(0);
 5810   format %{ "[$reg, $off]" %}
 5811   interface(MEMORY_INTER) %{
 5812     base($reg);
 5813     index(0xffffffff);
 5814     scale(0x0);
 5815     disp($off);
 5816   %}
 5817 %}
 5818 
 5819 operand indOffL(iRegP reg, immLoffset off)
 5820 %{
 5821   constraint(ALLOC_IN_RC(ptr_reg));
 5822   match(AddP reg off);
 5823   op_cost(0);
 5824   format %{ "[$reg, $off]" %}
 5825   interface(MEMORY_INTER) %{
 5826     base($reg);
 5827     index(0xffffffff);
 5828     scale(0x0);
 5829     disp($off);
 5830   %}
 5831 %}
 5832 
 5833 operand indOffL1(iRegP reg, immLoffset1 off)
 5834 %{
 5835   constraint(ALLOC_IN_RC(ptr_reg));
 5836   match(AddP reg off);
 5837   op_cost(0);
 5838   format %{ "[$reg, $off]" %}
 5839   interface(MEMORY_INTER) %{
 5840     base($reg);
 5841     index(0xffffffff);
 5842     scale(0x0);
 5843     disp($off);
 5844   %}
 5845 %}
 5846 
 5847 operand indOffL2(iRegP reg, immLoffset2 off)
 5848 %{
 5849   constraint(ALLOC_IN_RC(ptr_reg));
 5850   match(AddP reg off);
 5851   op_cost(0);
 5852   format %{ "[$reg, $off]" %}
 5853   interface(MEMORY_INTER) %{
 5854     base($reg);
 5855     index(0xffffffff);
 5856     scale(0x0);
 5857     disp($off);
 5858   %}
 5859 %}
 5860 
 5861 operand indOffL4(iRegP reg, immLoffset4 off)
 5862 %{
 5863   constraint(ALLOC_IN_RC(ptr_reg));
 5864   match(AddP reg off);
 5865   op_cost(0);
 5866   format %{ "[$reg, $off]" %}
 5867   interface(MEMORY_INTER) %{
 5868     base($reg);
 5869     index(0xffffffff);
 5870     scale(0x0);
 5871     disp($off);
 5872   %}
 5873 %}
 5874 
 5875 operand indOffL8(iRegP reg, immLoffset8 off)
 5876 %{
 5877   constraint(ALLOC_IN_RC(ptr_reg));
 5878   match(AddP reg off);
 5879   op_cost(0);
 5880   format %{ "[$reg, $off]" %}
 5881   interface(MEMORY_INTER) %{
 5882     base($reg);
 5883     index(0xffffffff);
 5884     scale(0x0);
 5885     disp($off);
 5886   %}
 5887 %}
 5888 
 5889 operand indOffL16(iRegP reg, immLoffset16 off)
 5890 %{
 5891   constraint(ALLOC_IN_RC(ptr_reg));
 5892   match(AddP reg off);
 5893   op_cost(0);
 5894   format %{ "[$reg, $off]" %}
 5895   interface(MEMORY_INTER) %{
 5896     base($reg);
 5897     index(0xffffffff);
 5898     scale(0x0);
 5899     disp($off);
 5900   %}
 5901 %}
 5902 
 5903 operand indirectN(iRegN reg)
 5904 %{
 5905   predicate(CompressedOops::shift() == 0);
 5906   constraint(ALLOC_IN_RC(ptr_reg));
 5907   match(DecodeN reg);
 5908   op_cost(0);
 5909   format %{ "[$reg]\t# narrow" %}
 5910   interface(MEMORY_INTER) %{
 5911     base($reg);
 5912     index(0xffffffff);
 5913     scale(0x0);
 5914     disp(0x0);
 5915   %}
 5916 %}
 5917 
 5918 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5919 %{
 5920   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5921   constraint(ALLOC_IN_RC(ptr_reg));
 5922   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5923   op_cost(0);
 5924   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5925   interface(MEMORY_INTER) %{
 5926     base($reg);
 5927     index($ireg);
 5928     scale($scale);
 5929     disp(0x0);
 5930   %}
 5931 %}
 5932 
 5933 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5934 %{
 5935   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5936   constraint(ALLOC_IN_RC(ptr_reg));
 5937   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5938   op_cost(0);
 5939   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5940   interface(MEMORY_INTER) %{
 5941     base($reg);
 5942     index($lreg);
 5943     scale($scale);
 5944     disp(0x0);
 5945   %}
 5946 %}
 5947 
 5948 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5949 %{
 5950   predicate(CompressedOops::shift() == 0);
 5951   constraint(ALLOC_IN_RC(ptr_reg));
 5952   match(AddP (DecodeN reg) (ConvI2L ireg));
 5953   op_cost(0);
 5954   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5955   interface(MEMORY_INTER) %{
 5956     base($reg);
 5957     index($ireg);
 5958     scale(0x0);
 5959     disp(0x0);
 5960   %}
 5961 %}
 5962 
 5963 operand indIndexN(iRegN reg, iRegL lreg)
 5964 %{
 5965   predicate(CompressedOops::shift() == 0);
 5966   constraint(ALLOC_IN_RC(ptr_reg));
 5967   match(AddP (DecodeN reg) lreg);
 5968   op_cost(0);
 5969   format %{ "$reg, $lreg\t# narrow" %}
 5970   interface(MEMORY_INTER) %{
 5971     base($reg);
 5972     index($lreg);
 5973     scale(0x0);
 5974     disp(0x0);
 5975   %}
 5976 %}
 5977 
 5978 operand indOffIN(iRegN reg, immIOffset off)
 5979 %{
 5980   predicate(CompressedOops::shift() == 0);
 5981   constraint(ALLOC_IN_RC(ptr_reg));
 5982   match(AddP (DecodeN reg) off);
 5983   op_cost(0);
 5984   format %{ "[$reg, $off]\t# narrow" %}
 5985   interface(MEMORY_INTER) %{
 5986     base($reg);
 5987     index(0xffffffff);
 5988     scale(0x0);
 5989     disp($off);
 5990   %}
 5991 %}
 5992 
 5993 operand indOffLN(iRegN reg, immLoffset off)
 5994 %{
 5995   predicate(CompressedOops::shift() == 0);
 5996   constraint(ALLOC_IN_RC(ptr_reg));
 5997   match(AddP (DecodeN reg) off);
 5998   op_cost(0);
 5999   format %{ "[$reg, $off]\t# narrow" %}
 6000   interface(MEMORY_INTER) %{
 6001     base($reg);
 6002     index(0xffffffff);
 6003     scale(0x0);
 6004     disp($off);
 6005   %}
 6006 %}
 6007 
 6008 
 6009 
 6010 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6011 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6012 %{
 6013   constraint(ALLOC_IN_RC(ptr_reg));
 6014   match(AddP reg off);
 6015   op_cost(0);
 6016   format %{ "[$reg, $off]" %}
 6017   interface(MEMORY_INTER) %{
 6018     base($reg);
 6019     index(0xffffffff);
 6020     scale(0x0);
 6021     disp($off);
 6022   %}
 6023 %}
 6024 
 6025 //----------Special Memory Operands--------------------------------------------
 6026 // Stack Slot Operand - This operand is used for loading and storing temporary
 6027 //                      values on the stack where a match requires a value to
 6028 //                      flow through memory.
 6029 operand stackSlotP(sRegP reg)
 6030 %{
 6031   constraint(ALLOC_IN_RC(stack_slots));
 6032   op_cost(100);
 6033   // No match rule because this operand is only generated in matching
 6034   // match(RegP);
 6035   format %{ "[$reg]" %}
 6036   interface(MEMORY_INTER) %{
 6037     base(0x1e);  // RSP
 6038     index(0x0);  // No Index
 6039     scale(0x0);  // No Scale
 6040     disp($reg);  // Stack Offset
 6041   %}
 6042 %}
 6043 
 6044 operand stackSlotI(sRegI reg)
 6045 %{
 6046   constraint(ALLOC_IN_RC(stack_slots));
 6047   // No match rule because this operand is only generated in matching
 6048   // match(RegI);
 6049   format %{ "[$reg]" %}
 6050   interface(MEMORY_INTER) %{
 6051     base(0x1e);  // RSP
 6052     index(0x0);  // No Index
 6053     scale(0x0);  // No Scale
 6054     disp($reg);  // Stack Offset
 6055   %}
 6056 %}
 6057 
 6058 operand stackSlotF(sRegF reg)
 6059 %{
 6060   constraint(ALLOC_IN_RC(stack_slots));
 6061   // No match rule because this operand is only generated in matching
 6062   // match(RegF);
 6063   format %{ "[$reg]" %}
 6064   interface(MEMORY_INTER) %{
 6065     base(0x1e);  // RSP
 6066     index(0x0);  // No Index
 6067     scale(0x0);  // No Scale
 6068     disp($reg);  // Stack Offset
 6069   %}
 6070 %}
 6071 
 6072 operand stackSlotD(sRegD reg)
 6073 %{
 6074   constraint(ALLOC_IN_RC(stack_slots));
 6075   // No match rule because this operand is only generated in matching
 6076   // match(RegD);
 6077   format %{ "[$reg]" %}
 6078   interface(MEMORY_INTER) %{
 6079     base(0x1e);  // RSP
 6080     index(0x0);  // No Index
 6081     scale(0x0);  // No Scale
 6082     disp($reg);  // Stack Offset
 6083   %}
 6084 %}
 6085 
 6086 operand stackSlotL(sRegL reg)
 6087 %{
 6088   constraint(ALLOC_IN_RC(stack_slots));
 6089   // No match rule because this operand is only generated in matching
 6090   // match(RegL);
 6091   format %{ "[$reg]" %}
 6092   interface(MEMORY_INTER) %{
 6093     base(0x1e);  // RSP
 6094     index(0x0);  // No Index
 6095     scale(0x0);  // No Scale
 6096     disp($reg);  // Stack Offset
 6097   %}
 6098 %}
 6099 
 6100 // Operands for expressing Control Flow
 6101 // NOTE: Label is a predefined operand which should not be redefined in
 6102 //       the AD file. It is generically handled within the ADLC.
 6103 
 6104 //----------Conditional Branch Operands----------------------------------------
 6105 // Comparison Op  - This is the operation of the comparison, and is limited to
 6106 //                  the following set of codes:
 6107 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6108 //
 6109 // Other attributes of the comparison, such as unsignedness, are specified
 6110 // by the comparison instruction that sets a condition code flags register.
 6111 // That result is represented by a flags operand whose subtype is appropriate
 6112 // to the unsignedness (etc.) of the comparison.
 6113 //
 6114 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6115 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6116 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6117 
 6118 // used for signed integral comparisons and fp comparisons
 6119 
 6120 operand cmpOp()
 6121 %{
 6122   match(Bool);
 6123 
 6124   format %{ "" %}
 6125   interface(COND_INTER) %{
 6126     equal(0x0, "eq");
 6127     not_equal(0x1, "ne");
 6128     less(0xb, "lt");
 6129     greater_equal(0xa, "ge");
 6130     less_equal(0xd, "le");
 6131     greater(0xc, "gt");
 6132     overflow(0x6, "vs");
 6133     no_overflow(0x7, "vc");
 6134   %}
 6135 %}
 6136 
 6137 // used for unsigned integral comparisons
 6138 
 6139 operand cmpOpU()
 6140 %{
 6141   match(Bool);
 6142 
 6143   format %{ "" %}
 6144   interface(COND_INTER) %{
 6145     equal(0x0, "eq");
 6146     not_equal(0x1, "ne");
 6147     less(0x3, "lo");
 6148     greater_equal(0x2, "hs");
 6149     less_equal(0x9, "ls");
 6150     greater(0x8, "hi");
 6151     overflow(0x6, "vs");
 6152     no_overflow(0x7, "vc");
 6153   %}
 6154 %}
 6155 
 6156 // used for certain integral comparisons which can be
 6157 // converted to cbxx or tbxx instructions
 6158 
 6159 operand cmpOpEqNe()
 6160 %{
 6161   match(Bool);
 6162   op_cost(0);
 6163   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6164             || n->as_Bool()->_test._test == BoolTest::eq);
 6165 
 6166   format %{ "" %}
 6167   interface(COND_INTER) %{
 6168     equal(0x0, "eq");
 6169     not_equal(0x1, "ne");
 6170     less(0xb, "lt");
 6171     greater_equal(0xa, "ge");
 6172     less_equal(0xd, "le");
 6173     greater(0xc, "gt");
 6174     overflow(0x6, "vs");
 6175     no_overflow(0x7, "vc");
 6176   %}
 6177 %}
 6178 
 6179 // used for certain integral comparisons which can be
 6180 // converted to cbxx or tbxx instructions
 6181 
 6182 operand cmpOpLtGe()
 6183 %{
 6184   match(Bool);
 6185   op_cost(0);
 6186 
 6187   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6188             || n->as_Bool()->_test._test == BoolTest::ge);
 6189 
 6190   format %{ "" %}
 6191   interface(COND_INTER) %{
 6192     equal(0x0, "eq");
 6193     not_equal(0x1, "ne");
 6194     less(0xb, "lt");
 6195     greater_equal(0xa, "ge");
 6196     less_equal(0xd, "le");
 6197     greater(0xc, "gt");
 6198     overflow(0x6, "vs");
 6199     no_overflow(0x7, "vc");
 6200   %}
 6201 %}
 6202 
 6203 // used for certain unsigned integral comparisons which can be
 6204 // converted to cbxx or tbxx instructions
 6205 
 6206 operand cmpOpUEqNeLtGe()
 6207 %{
 6208   match(Bool);
 6209   op_cost(0);
 6210 
 6211   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6212             || n->as_Bool()->_test._test == BoolTest::ne
 6213             || n->as_Bool()->_test._test == BoolTest::lt
 6214             || n->as_Bool()->_test._test == BoolTest::ge);
 6215 
 6216   format %{ "" %}
 6217   interface(COND_INTER) %{
 6218     equal(0x0, "eq");
 6219     not_equal(0x1, "ne");
 6220     less(0xb, "lt");
 6221     greater_equal(0xa, "ge");
 6222     less_equal(0xd, "le");
 6223     greater(0xc, "gt");
 6224     overflow(0x6, "vs");
 6225     no_overflow(0x7, "vc");
 6226   %}
 6227 %}
 6228 
 6229 // Special operand allowing long args to int ops to be truncated for free
 6230 
 6231 operand iRegL2I(iRegL reg) %{
 6232 
 6233   op_cost(0);
 6234 
 6235   match(ConvL2I reg);
 6236 
 6237   format %{ "l2i($reg)" %}
 6238 
 6239   interface(REG_INTER)
 6240 %}
 6241 
 6242 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6243 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6244 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6245 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6246 
 6247 //----------OPERAND CLASSES----------------------------------------------------
 6248 // Operand Classes are groups of operands that are used as to simplify
 6249 // instruction definitions by not requiring the AD writer to specify
 6250 // separate instructions for every form of operand when the
 6251 // instruction accepts multiple operand types with the same basic
 6252 // encoding and format. The classic case of this is memory operands.
 6253 
 6254 // memory is used to define read/write location for load/store
 6255 // instruction defs. we can turn a memory op into an Address
 6256 
 6257 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6258                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6259 
 6260 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6261                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6262 
 6263 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6264                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6265 
 6266 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6267                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6268 
 6269 // All of the memory operands. For the pipeline description.
 6270 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6271                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6272                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6273 
 6274 
 6275 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6276 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6277 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6278 // can be elided because the 32-bit instruction will just employ the
 6279 // lower 32 bits anyway.
 6280 //
 6281 // n.b. this does not elide all L2I conversions. if the truncated
 6282 // value is consumed by more than one operation then the ConvL2I
 6283 // cannot be bundled into the consuming nodes so an l2i gets planted
 6284 // (actually a movw $dst $src) and the downstream instructions consume
 6285 // the result of the l2i as an iRegI input. That's a shame since the
 6286 // movw is actually redundant but its not too costly.
 6287 
 6288 opclass iRegIorL2I(iRegI, iRegL2I);
 6289 
 6290 //----------PIPELINE-----------------------------------------------------------
 6291 // Rules which define the behavior of the target architectures pipeline.
 6292 
 6293 // For specific pipelines, eg A53, define the stages of that pipeline
 6294 //pipe_desc(ISS, EX1, EX2, WR);
 6295 #define ISS S0
 6296 #define EX1 S1
 6297 #define EX2 S2
 6298 #define WR  S3
 6299 
 6300 // Integer ALU reg operation
 6301 pipeline %{
 6302 
 6303 attributes %{
 6304   // ARM instructions are of fixed length
 6305   fixed_size_instructions;        // Fixed size instructions TODO does
 6306   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6307   // ARM instructions come in 32-bit word units
 6308   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6309   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6310   instruction_fetch_units = 1;       // of 64 bytes
 6311 
 6312   // List of nop instructions
 6313   nops( MachNop );
 6314 %}
 6315 
 6316 // We don't use an actual pipeline model so don't care about resources
 6317 // or description. we do use pipeline classes to introduce fixed
 6318 // latencies
 6319 
 6320 //----------RESOURCES----------------------------------------------------------
 6321 // Resources are the functional units available to the machine
 6322 
 6323 resources( INS0, INS1, INS01 = INS0 | INS1,
 6324            ALU0, ALU1, ALU = ALU0 | ALU1,
 6325            MAC,
 6326            DIV,
 6327            BRANCH,
 6328            LDST,
 6329            NEON_FP);
 6330 
 6331 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6332 // Pipeline Description specifies the stages in the machine's pipeline
 6333 
 6334 // Define the pipeline as a generic 6 stage pipeline
 6335 pipe_desc(S0, S1, S2, S3, S4, S5);
 6336 
 6337 //----------PIPELINE CLASSES---------------------------------------------------
 6338 // Pipeline Classes describe the stages in which input and output are
 6339 // referenced by the hardware pipeline.
 6340 
 6341 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6342 %{
 6343   single_instruction;
 6344   src1   : S1(read);
 6345   src2   : S2(read);
 6346   dst    : S5(write);
 6347   INS01  : ISS;
 6348   NEON_FP : S5;
 6349 %}
 6350 
 6351 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6352 %{
 6353   single_instruction;
 6354   src1   : S1(read);
 6355   src2   : S2(read);
 6356   dst    : S5(write);
 6357   INS01  : ISS;
 6358   NEON_FP : S5;
 6359 %}
 6360 
 6361 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6362 %{
 6363   single_instruction;
 6364   src    : S1(read);
 6365   dst    : S5(write);
 6366   INS01  : ISS;
 6367   NEON_FP : S5;
 6368 %}
 6369 
 6370 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6371 %{
 6372   single_instruction;
 6373   src    : S1(read);
 6374   dst    : S5(write);
 6375   INS01  : ISS;
 6376   NEON_FP : S5;
 6377 %}
 6378 
 6379 pipe_class fp_d2f(vRegF dst, vRegD src)
 6380 %{
 6381   single_instruction;
 6382   src    : S1(read);
 6383   dst    : S5(write);
 6384   INS01  : ISS;
 6385   NEON_FP : S5;
 6386 %}
 6387 
 6388 pipe_class fp_f2d(vRegD dst, vRegF src)
 6389 %{
 6390   single_instruction;
 6391   src    : S1(read);
 6392   dst    : S5(write);
 6393   INS01  : ISS;
 6394   NEON_FP : S5;
 6395 %}
 6396 
 6397 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6398 %{
 6399   single_instruction;
 6400   src    : S1(read);
 6401   dst    : S5(write);
 6402   INS01  : ISS;
 6403   NEON_FP : S5;
 6404 %}
 6405 
 6406 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6407 %{
 6408   single_instruction;
 6409   src    : S1(read);
 6410   dst    : S5(write);
 6411   INS01  : ISS;
 6412   NEON_FP : S5;
 6413 %}
 6414 
 6415 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6416 %{
 6417   single_instruction;
 6418   src    : S1(read);
 6419   dst    : S5(write);
 6420   INS01  : ISS;
 6421   NEON_FP : S5;
 6422 %}
 6423 
 6424 pipe_class fp_l2f(vRegF dst, iRegL src)
 6425 %{
 6426   single_instruction;
 6427   src    : S1(read);
 6428   dst    : S5(write);
 6429   INS01  : ISS;
 6430   NEON_FP : S5;
 6431 %}
 6432 
 6433 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6434 %{
 6435   single_instruction;
 6436   src    : S1(read);
 6437   dst    : S5(write);
 6438   INS01  : ISS;
 6439   NEON_FP : S5;
 6440 %}
 6441 
 6442 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6443 %{
 6444   single_instruction;
 6445   src    : S1(read);
 6446   dst    : S5(write);
 6447   INS01  : ISS;
 6448   NEON_FP : S5;
 6449 %}
 6450 
 6451 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6452 %{
 6453   single_instruction;
 6454   src    : S1(read);
 6455   dst    : S5(write);
 6456   INS01  : ISS;
 6457   NEON_FP : S5;
 6458 %}
 6459 
 6460 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6461 %{
 6462   single_instruction;
 6463   src    : S1(read);
 6464   dst    : S5(write);
 6465   INS01  : ISS;
 6466   NEON_FP : S5;
 6467 %}
 6468 
 6469 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6470 %{
 6471   single_instruction;
 6472   src1   : S1(read);
 6473   src2   : S2(read);
 6474   dst    : S5(write);
 6475   INS0   : ISS;
 6476   NEON_FP : S5;
 6477 %}
 6478 
 6479 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6480 %{
 6481   single_instruction;
 6482   src1   : S1(read);
 6483   src2   : S2(read);
 6484   dst    : S5(write);
 6485   INS0   : ISS;
 6486   NEON_FP : S5;
 6487 %}
 6488 
 6489 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6490 %{
 6491   single_instruction;
 6492   cr     : S1(read);
 6493   src1   : S1(read);
 6494   src2   : S1(read);
 6495   dst    : S3(write);
 6496   INS01  : ISS;
 6497   NEON_FP : S3;
 6498 %}
 6499 
 6500 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6501 %{
 6502   single_instruction;
 6503   cr     : S1(read);
 6504   src1   : S1(read);
 6505   src2   : S1(read);
 6506   dst    : S3(write);
 6507   INS01  : ISS;
 6508   NEON_FP : S3;
 6509 %}
 6510 
 6511 pipe_class fp_imm_s(vRegF dst)
 6512 %{
 6513   single_instruction;
 6514   dst    : S3(write);
 6515   INS01  : ISS;
 6516   NEON_FP : S3;
 6517 %}
 6518 
 6519 pipe_class fp_imm_d(vRegD dst)
 6520 %{
 6521   single_instruction;
 6522   dst    : S3(write);
 6523   INS01  : ISS;
 6524   NEON_FP : S3;
 6525 %}
 6526 
 6527 pipe_class fp_load_constant_s(vRegF dst)
 6528 %{
 6529   single_instruction;
 6530   dst    : S4(write);
 6531   INS01  : ISS;
 6532   NEON_FP : S4;
 6533 %}
 6534 
 6535 pipe_class fp_load_constant_d(vRegD dst)
 6536 %{
 6537   single_instruction;
 6538   dst    : S4(write);
 6539   INS01  : ISS;
 6540   NEON_FP : S4;
 6541 %}
 6542 
 6543 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6544 %{
 6545   single_instruction;
 6546   dst    : S5(write);
 6547   src1   : S1(read);
 6548   src2   : S1(read);
 6549   INS01  : ISS;
 6550   NEON_FP : S5;
 6551 %}
 6552 
 6553 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6554 %{
 6555   single_instruction;
 6556   dst    : S5(write);
 6557   src1   : S1(read);
 6558   src2   : S1(read);
 6559   INS0   : ISS;
 6560   NEON_FP : S5;
 6561 %}
 6562 
 6563 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6564 %{
 6565   single_instruction;
 6566   dst    : S5(write);
 6567   src1   : S1(read);
 6568   src2   : S1(read);
 6569   dst    : S1(read);
 6570   INS01  : ISS;
 6571   NEON_FP : S5;
 6572 %}
 6573 
 6574 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6575 %{
 6576   single_instruction;
 6577   dst    : S5(write);
 6578   src1   : S1(read);
 6579   src2   : S1(read);
 6580   dst    : S1(read);
 6581   INS0   : ISS;
 6582   NEON_FP : S5;
 6583 %}
 6584 
 6585 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6586 %{
 6587   single_instruction;
 6588   dst    : S4(write);
 6589   src1   : S2(read);
 6590   src2   : S2(read);
 6591   INS01  : ISS;
 6592   NEON_FP : S4;
 6593 %}
 6594 
 6595 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6596 %{
 6597   single_instruction;
 6598   dst    : S4(write);
 6599   src1   : S2(read);
 6600   src2   : S2(read);
 6601   INS0   : ISS;
 6602   NEON_FP : S4;
 6603 %}
 6604 
 6605 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6606 %{
 6607   single_instruction;
 6608   dst    : S3(write);
 6609   src1   : S2(read);
 6610   src2   : S2(read);
 6611   INS01  : ISS;
 6612   NEON_FP : S3;
 6613 %}
 6614 
 6615 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6616 %{
 6617   single_instruction;
 6618   dst    : S3(write);
 6619   src1   : S2(read);
 6620   src2   : S2(read);
 6621   INS0   : ISS;
 6622   NEON_FP : S3;
 6623 %}
 6624 
 6625 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6626 %{
 6627   single_instruction;
 6628   dst    : S3(write);
 6629   src    : S1(read);
 6630   shift  : S1(read);
 6631   INS01  : ISS;
 6632   NEON_FP : S3;
 6633 %}
 6634 
 6635 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6636 %{
 6637   single_instruction;
 6638   dst    : S3(write);
 6639   src    : S1(read);
 6640   shift  : S1(read);
 6641   INS0   : ISS;
 6642   NEON_FP : S3;
 6643 %}
 6644 
 6645 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6646 %{
 6647   single_instruction;
 6648   dst    : S3(write);
 6649   src    : S1(read);
 6650   INS01  : ISS;
 6651   NEON_FP : S3;
 6652 %}
 6653 
 6654 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6655 %{
 6656   single_instruction;
 6657   dst    : S3(write);
 6658   src    : S1(read);
 6659   INS0   : ISS;
 6660   NEON_FP : S3;
 6661 %}
 6662 
 6663 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6664 %{
 6665   single_instruction;
 6666   dst    : S5(write);
 6667   src1   : S1(read);
 6668   src2   : S1(read);
 6669   INS01  : ISS;
 6670   NEON_FP : S5;
 6671 %}
 6672 
 6673 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6674 %{
 6675   single_instruction;
 6676   dst    : S5(write);
 6677   src1   : S1(read);
 6678   src2   : S1(read);
 6679   INS0   : ISS;
 6680   NEON_FP : S5;
 6681 %}
 6682 
 6683 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6684 %{
 6685   single_instruction;
 6686   dst    : S5(write);
 6687   src1   : S1(read);
 6688   src2   : S1(read);
 6689   INS0   : ISS;
 6690   NEON_FP : S5;
 6691 %}
 6692 
 6693 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6694 %{
 6695   single_instruction;
 6696   dst    : S5(write);
 6697   src1   : S1(read);
 6698   src2   : S1(read);
 6699   INS0   : ISS;
 6700   NEON_FP : S5;
 6701 %}
 6702 
 6703 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6704 %{
 6705   single_instruction;
 6706   dst    : S5(write);
 6707   src    : S1(read);
 6708   INS0   : ISS;
 6709   NEON_FP : S5;
 6710 %}
 6711 
 6712 pipe_class vunop_fp64(vecD dst, vecD src)
 6713 %{
 6714   single_instruction;
 6715   dst    : S5(write);
 6716   src    : S1(read);
 6717   INS01  : ISS;
 6718   NEON_FP : S5;
 6719 %}
 6720 
 6721 pipe_class vunop_fp128(vecX dst, vecX src)
 6722 %{
 6723   single_instruction;
 6724   dst    : S5(write);
 6725   src    : S1(read);
 6726   INS0   : ISS;
 6727   NEON_FP : S5;
 6728 %}
 6729 
 6730 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6731 %{
 6732   single_instruction;
 6733   dst    : S3(write);
 6734   src    : S1(read);
 6735   INS01  : ISS;
 6736   NEON_FP : S3;
 6737 %}
 6738 
 6739 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6740 %{
 6741   single_instruction;
 6742   dst    : S3(write);
 6743   src    : S1(read);
 6744   INS01  : ISS;
 6745   NEON_FP : S3;
 6746 %}
 6747 
 6748 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6749 %{
 6750   single_instruction;
 6751   dst    : S3(write);
 6752   src    : S1(read);
 6753   INS01  : ISS;
 6754   NEON_FP : S3;
 6755 %}
 6756 
 6757 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6758 %{
 6759   single_instruction;
 6760   dst    : S3(write);
 6761   src    : S1(read);
 6762   INS01  : ISS;
 6763   NEON_FP : S3;
 6764 %}
 6765 
 6766 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6767 %{
 6768   single_instruction;
 6769   dst    : S3(write);
 6770   src    : S1(read);
 6771   INS01  : ISS;
 6772   NEON_FP : S3;
 6773 %}
 6774 
 6775 pipe_class vmovi_reg_imm64(vecD dst)
 6776 %{
 6777   single_instruction;
 6778   dst    : S3(write);
 6779   INS01  : ISS;
 6780   NEON_FP : S3;
 6781 %}
 6782 
 6783 pipe_class vmovi_reg_imm128(vecX dst)
 6784 %{
 6785   single_instruction;
 6786   dst    : S3(write);
 6787   INS0   : ISS;
 6788   NEON_FP : S3;
 6789 %}
 6790 
 6791 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6792 %{
 6793   single_instruction;
 6794   dst    : S5(write);
 6795   mem    : ISS(read);
 6796   INS01  : ISS;
 6797   NEON_FP : S3;
 6798 %}
 6799 
 6800 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6801 %{
 6802   single_instruction;
 6803   dst    : S5(write);
 6804   mem    : ISS(read);
 6805   INS01  : ISS;
 6806   NEON_FP : S3;
 6807 %}
 6808 
 6809 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6810 %{
 6811   single_instruction;
 6812   mem    : ISS(read);
 6813   src    : S2(read);
 6814   INS01  : ISS;
 6815   NEON_FP : S3;
 6816 %}
 6817 
 6818 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6819 %{
 6820   single_instruction;
 6821   mem    : ISS(read);
 6822   src    : S2(read);
 6823   INS01  : ISS;
 6824   NEON_FP : S3;
 6825 %}
 6826 
 6827 //------- Integer ALU operations --------------------------
 6828 
 6829 // Integer ALU reg-reg operation
 6830 // Operands needed in EX1, result generated in EX2
 6831 // Eg.  ADD     x0, x1, x2
 6832 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6833 %{
 6834   single_instruction;
 6835   dst    : EX2(write);
 6836   src1   : EX1(read);
 6837   src2   : EX1(read);
 6838   INS01  : ISS; // Dual issue as instruction 0 or 1
 6839   ALU    : EX2;
 6840 %}
 6841 
 6842 // Integer ALU reg-reg operation with constant shift
 6843 // Shifted register must be available in LATE_ISS instead of EX1
 6844 // Eg.  ADD     x0, x1, x2, LSL #2
 6845 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6846 %{
 6847   single_instruction;
 6848   dst    : EX2(write);
 6849   src1   : EX1(read);
 6850   src2   : ISS(read);
 6851   INS01  : ISS;
 6852   ALU    : EX2;
 6853 %}
 6854 
 6855 // Integer ALU reg operation with constant shift
 6856 // Eg.  LSL     x0, x1, #shift
 6857 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6858 %{
 6859   single_instruction;
 6860   dst    : EX2(write);
 6861   src1   : ISS(read);
 6862   INS01  : ISS;
 6863   ALU    : EX2;
 6864 %}
 6865 
 6866 // Integer ALU reg-reg operation with variable shift
 6867 // Both operands must be available in LATE_ISS instead of EX1
 6868 // Result is available in EX1 instead of EX2
 6869 // Eg.  LSLV    x0, x1, x2
 6870 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6871 %{
 6872   single_instruction;
 6873   dst    : EX1(write);
 6874   src1   : ISS(read);
 6875   src2   : ISS(read);
 6876   INS01  : ISS;
 6877   ALU    : EX1;
 6878 %}
 6879 
 6880 // Integer ALU reg-reg operation with extract
 6881 // As for _vshift above, but result generated in EX2
 6882 // Eg.  EXTR    x0, x1, x2, #N
 6883 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6884 %{
 6885   single_instruction;
 6886   dst    : EX2(write);
 6887   src1   : ISS(read);
 6888   src2   : ISS(read);
 6889   INS1   : ISS; // Can only dual issue as Instruction 1
 6890   ALU    : EX1;
 6891 %}
 6892 
 6893 // Integer ALU reg operation
 6894 // Eg.  NEG     x0, x1
 6895 pipe_class ialu_reg(iRegI dst, iRegI src)
 6896 %{
 6897   single_instruction;
 6898   dst    : EX2(write);
 6899   src    : EX1(read);
 6900   INS01  : ISS;
 6901   ALU    : EX2;
 6902 %}
 6903 
 6904 // Integer ALU reg mmediate operation
 6905 // Eg.  ADD     x0, x1, #N
 6906 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6907 %{
 6908   single_instruction;
 6909   dst    : EX2(write);
 6910   src1   : EX1(read);
 6911   INS01  : ISS;
 6912   ALU    : EX2;
 6913 %}
 6914 
 6915 // Integer ALU immediate operation (no source operands)
 6916 // Eg.  MOV     x0, #N
 6917 pipe_class ialu_imm(iRegI dst)
 6918 %{
 6919   single_instruction;
 6920   dst    : EX1(write);
 6921   INS01  : ISS;
 6922   ALU    : EX1;
 6923 %}
 6924 
 6925 //------- Compare operation -------------------------------
 6926 
 6927 // Compare reg-reg
 6928 // Eg.  CMP     x0, x1
 6929 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6930 %{
 6931   single_instruction;
 6932 //  fixed_latency(16);
 6933   cr     : EX2(write);
 6934   op1    : EX1(read);
 6935   op2    : EX1(read);
 6936   INS01  : ISS;
 6937   ALU    : EX2;
 6938 %}
 6939 
 6940 // Compare reg-reg
 6941 // Eg.  CMP     x0, #N
 6942 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6943 %{
 6944   single_instruction;
 6945 //  fixed_latency(16);
 6946   cr     : EX2(write);
 6947   op1    : EX1(read);
 6948   INS01  : ISS;
 6949   ALU    : EX2;
 6950 %}
 6951 
 6952 //------- Conditional instructions ------------------------
 6953 
 6954 // Conditional no operands
 6955 // Eg.  CSINC   x0, zr, zr, <cond>
 6956 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6957 %{
 6958   single_instruction;
 6959   cr     : EX1(read);
 6960   dst    : EX2(write);
 6961   INS01  : ISS;
 6962   ALU    : EX2;
 6963 %}
 6964 
 6965 // Conditional 2 operand
 6966 // EG.  CSEL    X0, X1, X2, <cond>
 6967 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6968 %{
 6969   single_instruction;
 6970   cr     : EX1(read);
 6971   src1   : EX1(read);
 6972   src2   : EX1(read);
 6973   dst    : EX2(write);
 6974   INS01  : ISS;
 6975   ALU    : EX2;
 6976 %}
 6977 
 6978 // Conditional 2 operand
 6979 // EG.  CSEL    X0, X1, X2, <cond>
 6980 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6981 %{
 6982   single_instruction;
 6983   cr     : EX1(read);
 6984   src    : EX1(read);
 6985   dst    : EX2(write);
 6986   INS01  : ISS;
 6987   ALU    : EX2;
 6988 %}
 6989 
 6990 //------- Multiply pipeline operations --------------------
 6991 
 6992 // Multiply reg-reg
 6993 // Eg.  MUL     w0, w1, w2
 6994 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6995 %{
 6996   single_instruction;
 6997   dst    : WR(write);
 6998   src1   : ISS(read);
 6999   src2   : ISS(read);
 7000   INS01  : ISS;
 7001   MAC    : WR;
 7002 %}
 7003 
 7004 // Multiply accumulate
 7005 // Eg.  MADD    w0, w1, w2, w3
 7006 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7007 %{
 7008   single_instruction;
 7009   dst    : WR(write);
 7010   src1   : ISS(read);
 7011   src2   : ISS(read);
 7012   src3   : ISS(read);
 7013   INS01  : ISS;
 7014   MAC    : WR;
 7015 %}
 7016 
 7017 // Eg.  MUL     w0, w1, w2
 7018 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7019 %{
 7020   single_instruction;
 7021   fixed_latency(3); // Maximum latency for 64 bit mul
 7022   dst    : WR(write);
 7023   src1   : ISS(read);
 7024   src2   : ISS(read);
 7025   INS01  : ISS;
 7026   MAC    : WR;
 7027 %}
 7028 
 7029 // Multiply accumulate
 7030 // Eg.  MADD    w0, w1, w2, w3
 7031 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7032 %{
 7033   single_instruction;
 7034   fixed_latency(3); // Maximum latency for 64 bit mul
 7035   dst    : WR(write);
 7036   src1   : ISS(read);
 7037   src2   : ISS(read);
 7038   src3   : ISS(read);
 7039   INS01  : ISS;
 7040   MAC    : WR;
 7041 %}
 7042 
 7043 //------- Divide pipeline operations --------------------
 7044 
 7045 // Eg.  SDIV    w0, w1, w2
 7046 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7047 %{
 7048   single_instruction;
 7049   fixed_latency(8); // Maximum latency for 32 bit divide
 7050   dst    : WR(write);
 7051   src1   : ISS(read);
 7052   src2   : ISS(read);
 7053   INS0   : ISS; // Can only dual issue as instruction 0
 7054   DIV    : WR;
 7055 %}
 7056 
 7057 // Eg.  SDIV    x0, x1, x2
 7058 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7059 %{
 7060   single_instruction;
 7061   fixed_latency(16); // Maximum latency for 64 bit divide
 7062   dst    : WR(write);
 7063   src1   : ISS(read);
 7064   src2   : ISS(read);
 7065   INS0   : ISS; // Can only dual issue as instruction 0
 7066   DIV    : WR;
 7067 %}
 7068 
 7069 //------- Load pipeline operations ------------------------
 7070 
 7071 // Load - prefetch
 7072 // Eg.  PFRM    <mem>
 7073 pipe_class iload_prefetch(memory mem)
 7074 %{
 7075   single_instruction;
 7076   mem    : ISS(read);
 7077   INS01  : ISS;
 7078   LDST   : WR;
 7079 %}
 7080 
 7081 // Load - reg, mem
 7082 // Eg.  LDR     x0, <mem>
 7083 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7084 %{
 7085   single_instruction;
 7086   dst    : WR(write);
 7087   mem    : ISS(read);
 7088   INS01  : ISS;
 7089   LDST   : WR;
 7090 %}
 7091 
 7092 // Load - reg, reg
 7093 // Eg.  LDR     x0, [sp, x1]
 7094 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7095 %{
 7096   single_instruction;
 7097   dst    : WR(write);
 7098   src    : ISS(read);
 7099   INS01  : ISS;
 7100   LDST   : WR;
 7101 %}
 7102 
 7103 //------- Store pipeline operations -----------------------
 7104 
 7105 // Store - zr, mem
 7106 // Eg.  STR     zr, <mem>
 7107 pipe_class istore_mem(memory mem)
 7108 %{
 7109   single_instruction;
 7110   mem    : ISS(read);
 7111   INS01  : ISS;
 7112   LDST   : WR;
 7113 %}
 7114 
 7115 // Store - reg, mem
 7116 // Eg.  STR     x0, <mem>
 7117 pipe_class istore_reg_mem(iRegI src, memory mem)
 7118 %{
 7119   single_instruction;
 7120   mem    : ISS(read);
 7121   src    : EX2(read);
 7122   INS01  : ISS;
 7123   LDST   : WR;
 7124 %}
 7125 
 7126 // Store - reg, reg
 7127 // Eg. STR      x0, [sp, x1]
 7128 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7129 %{
 7130   single_instruction;
 7131   dst    : ISS(read);
 7132   src    : EX2(read);
 7133   INS01  : ISS;
 7134   LDST   : WR;
 7135 %}
 7136 
 7137 //------- Store pipeline operations -----------------------
 7138 
 7139 // Branch
 7140 pipe_class pipe_branch()
 7141 %{
 7142   single_instruction;
 7143   INS01  : ISS;
 7144   BRANCH : EX1;
 7145 %}
 7146 
 7147 // Conditional branch
 7148 pipe_class pipe_branch_cond(rFlagsReg cr)
 7149 %{
 7150   single_instruction;
 7151   cr     : EX1(read);
 7152   INS01  : ISS;
 7153   BRANCH : EX1;
 7154 %}
 7155 
 7156 // Compare & Branch
 7157 // EG.  CBZ/CBNZ
 7158 pipe_class pipe_cmp_branch(iRegI op1)
 7159 %{
 7160   single_instruction;
 7161   op1    : EX1(read);
 7162   INS01  : ISS;
 7163   BRANCH : EX1;
 7164 %}
 7165 
 7166 //------- Synchronisation operations ----------------------
 7167 
 7168 // Any operation requiring serialization.
 7169 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7170 pipe_class pipe_serial()
 7171 %{
 7172   single_instruction;
 7173   force_serialization;
 7174   fixed_latency(16);
 7175   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7176   LDST   : WR;
 7177 %}
 7178 
 7179 // Generic big/slow expanded idiom - also serialized
 7180 pipe_class pipe_slow()
 7181 %{
 7182   instruction_count(10);
 7183   multiple_bundles;
 7184   force_serialization;
 7185   fixed_latency(16);
 7186   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7187   LDST   : WR;
 7188 %}
 7189 
 7190 // Empty pipeline class
 7191 pipe_class pipe_class_empty()
 7192 %{
 7193   single_instruction;
 7194   fixed_latency(0);
 7195 %}
 7196 
 7197 // Default pipeline class.
 7198 pipe_class pipe_class_default()
 7199 %{
 7200   single_instruction;
 7201   fixed_latency(2);
 7202 %}
 7203 
 7204 // Pipeline class for compares.
 7205 pipe_class pipe_class_compare()
 7206 %{
 7207   single_instruction;
 7208   fixed_latency(16);
 7209 %}
 7210 
 7211 // Pipeline class for memory operations.
 7212 pipe_class pipe_class_memory()
 7213 %{
 7214   single_instruction;
 7215   fixed_latency(16);
 7216 %}
 7217 
 7218 // Pipeline class for call.
 7219 pipe_class pipe_class_call()
 7220 %{
 7221   single_instruction;
 7222   fixed_latency(100);
 7223 %}
 7224 
 7225 // Define the class for the Nop node.
 7226 define %{
 7227    MachNop = pipe_class_empty;
 7228 %}
 7229 
 7230 %}
 7231 //----------INSTRUCTIONS-------------------------------------------------------
 7232 //
 7233 // match      -- States which machine-independent subtree may be replaced
 7234 //               by this instruction.
 7235 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7236 //               selection to identify a minimum cost tree of machine
 7237 //               instructions that matches a tree of machine-independent
 7238 //               instructions.
 7239 // format     -- A string providing the disassembly for this instruction.
 7240 //               The value of an instruction's operand may be inserted
 7241 //               by referring to it with a '$' prefix.
 7242 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7243 //               to within an encode class as $primary, $secondary, and $tertiary
 7244 //               rrspectively.  The primary opcode is commonly used to
 7245 //               indicate the type of machine instruction, while secondary
 7246 //               and tertiary are often used for prefix options or addressing
 7247 //               modes.
 7248 // ins_encode -- A list of encode classes with parameters. The encode class
 7249 //               name must have been defined in an 'enc_class' specification
 7250 //               in the encode section of the architecture description.
 7251 
 7252 // ============================================================================
 7253 // Memory (Load/Store) Instructions
 7254 
 7255 // Load Instructions
 7256 
 7257 // Load Byte (8 bit signed)
 7258 instruct loadB(iRegINoSp dst, memory1 mem)
 7259 %{
 7260   match(Set dst (LoadB mem));
 7261   predicate(!needs_acquiring_load(n));
 7262 
 7263   ins_cost(4 * INSN_COST);
 7264   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7265 
 7266   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7267 
 7268   ins_pipe(iload_reg_mem);
 7269 %}
 7270 
 7271 // Load Byte (8 bit signed) into long
 7272 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7273 %{
 7274   match(Set dst (ConvI2L (LoadB mem)));
 7275   predicate(!needs_acquiring_load(n->in(1)));
 7276 
 7277   ins_cost(4 * INSN_COST);
 7278   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7279 
 7280   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7281 
 7282   ins_pipe(iload_reg_mem);
 7283 %}
 7284 
 7285 // Load Byte (8 bit unsigned)
 7286 instruct loadUB(iRegINoSp dst, memory1 mem)
 7287 %{
 7288   match(Set dst (LoadUB mem));
 7289   predicate(!needs_acquiring_load(n));
 7290 
 7291   ins_cost(4 * INSN_COST);
 7292   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7293 
 7294   ins_encode(aarch64_enc_ldrb(dst, mem));
 7295 
 7296   ins_pipe(iload_reg_mem);
 7297 %}
 7298 
 7299 // Load Byte (8 bit unsigned) into long
 7300 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7301 %{
 7302   match(Set dst (ConvI2L (LoadUB mem)));
 7303   predicate(!needs_acquiring_load(n->in(1)));
 7304 
 7305   ins_cost(4 * INSN_COST);
 7306   format %{ "ldrb  $dst, $mem\t# byte" %}
 7307 
 7308   ins_encode(aarch64_enc_ldrb(dst, mem));
 7309 
 7310   ins_pipe(iload_reg_mem);
 7311 %}
 7312 
 7313 // Load Short (16 bit signed)
 7314 instruct loadS(iRegINoSp dst, memory2 mem)
 7315 %{
 7316   match(Set dst (LoadS mem));
 7317   predicate(!needs_acquiring_load(n));
 7318 
 7319   ins_cost(4 * INSN_COST);
 7320   format %{ "ldrshw  $dst, $mem\t# short" %}
 7321 
 7322   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7323 
 7324   ins_pipe(iload_reg_mem);
 7325 %}
 7326 
 7327 // Load Short (16 bit signed) into long
 7328 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7329 %{
 7330   match(Set dst (ConvI2L (LoadS mem)));
 7331   predicate(!needs_acquiring_load(n->in(1)));
 7332 
 7333   ins_cost(4 * INSN_COST);
 7334   format %{ "ldrsh  $dst, $mem\t# short" %}
 7335 
 7336   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7337 
 7338   ins_pipe(iload_reg_mem);
 7339 %}
 7340 
 7341 // Load Char (16 bit unsigned)
 7342 instruct loadUS(iRegINoSp dst, memory2 mem)
 7343 %{
 7344   match(Set dst (LoadUS mem));
 7345   predicate(!needs_acquiring_load(n));
 7346 
 7347   ins_cost(4 * INSN_COST);
 7348   format %{ "ldrh  $dst, $mem\t# short" %}
 7349 
 7350   ins_encode(aarch64_enc_ldrh(dst, mem));
 7351 
 7352   ins_pipe(iload_reg_mem);
 7353 %}
 7354 
 7355 // Load Short/Char (16 bit unsigned) into long
 7356 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7357 %{
 7358   match(Set dst (ConvI2L (LoadUS mem)));
 7359   predicate(!needs_acquiring_load(n->in(1)));
 7360 
 7361   ins_cost(4 * INSN_COST);
 7362   format %{ "ldrh  $dst, $mem\t# short" %}
 7363 
 7364   ins_encode(aarch64_enc_ldrh(dst, mem));
 7365 
 7366   ins_pipe(iload_reg_mem);
 7367 %}
 7368 
 7369 // Load Integer (32 bit signed)
 7370 instruct loadI(iRegINoSp dst, memory4 mem)
 7371 %{
 7372   match(Set dst (LoadI mem));
 7373   predicate(!needs_acquiring_load(n));
 7374 
 7375   ins_cost(4 * INSN_COST);
 7376   format %{ "ldrw  $dst, $mem\t# int" %}
 7377 
 7378   ins_encode(aarch64_enc_ldrw(dst, mem));
 7379 
 7380   ins_pipe(iload_reg_mem);
 7381 %}
 7382 
 7383 // Load Integer (32 bit signed) into long
 7384 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7385 %{
 7386   match(Set dst (ConvI2L (LoadI mem)));
 7387   predicate(!needs_acquiring_load(n->in(1)));
 7388 
 7389   ins_cost(4 * INSN_COST);
 7390   format %{ "ldrsw  $dst, $mem\t# int" %}
 7391 
 7392   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7393 
 7394   ins_pipe(iload_reg_mem);
 7395 %}
 7396 
 7397 // Load Integer (32 bit unsigned) into long
 7398 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7399 %{
 7400   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7401   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7402 
 7403   ins_cost(4 * INSN_COST);
 7404   format %{ "ldrw  $dst, $mem\t# int" %}
 7405 
 7406   ins_encode(aarch64_enc_ldrw(dst, mem));
 7407 
 7408   ins_pipe(iload_reg_mem);
 7409 %}
 7410 
 7411 // Load Long (64 bit signed)
 7412 instruct loadL(iRegLNoSp dst, memory8 mem)
 7413 %{
 7414   match(Set dst (LoadL mem));
 7415   predicate(!needs_acquiring_load(n));
 7416 
 7417   ins_cost(4 * INSN_COST);
 7418   format %{ "ldr  $dst, $mem\t# int" %}
 7419 
 7420   ins_encode(aarch64_enc_ldr(dst, mem));
 7421 
 7422   ins_pipe(iload_reg_mem);
 7423 %}
 7424 
 7425 // Load Range
 7426 instruct loadRange(iRegINoSp dst, memory4 mem)
 7427 %{
 7428   match(Set dst (LoadRange mem));
 7429 
 7430   ins_cost(4 * INSN_COST);
 7431   format %{ "ldrw  $dst, $mem\t# range" %}
 7432 
 7433   ins_encode(aarch64_enc_ldrw(dst, mem));
 7434 
 7435   ins_pipe(iload_reg_mem);
 7436 %}
 7437 
 7438 // Load Pointer
 7439 instruct loadP(iRegPNoSp dst, memory8 mem)
 7440 %{
 7441   match(Set dst (LoadP mem));
 7442   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7443 
 7444   ins_cost(4 * INSN_COST);
 7445   format %{ "ldr  $dst, $mem\t# ptr" %}
 7446 
 7447   ins_encode(aarch64_enc_ldr(dst, mem));
 7448 
 7449   ins_pipe(iload_reg_mem);
 7450 %}
 7451 
 7452 // Load Compressed Pointer
 7453 instruct loadN(iRegNNoSp dst, memory4 mem)
 7454 %{
 7455   match(Set dst (LoadN mem));
 7456   predicate(!needs_acquiring_load(n));
 7457 
 7458   ins_cost(4 * INSN_COST);
 7459   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7460 
 7461   ins_encode(aarch64_enc_ldrw(dst, mem));
 7462 
 7463   ins_pipe(iload_reg_mem);
 7464 %}
 7465 
 7466 // Load Klass Pointer
 7467 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7468 %{
 7469   match(Set dst (LoadKlass mem));
 7470   predicate(!needs_acquiring_load(n));
 7471 
 7472   ins_cost(4 * INSN_COST);
 7473   format %{ "ldr  $dst, $mem\t# class" %}
 7474 
 7475   ins_encode(aarch64_enc_ldr(dst, mem));
 7476 
 7477   ins_pipe(iload_reg_mem);
 7478 %}
 7479 
 7480 // Load Narrow Klass Pointer
 7481 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7482 %{
 7483   match(Set dst (LoadNKlass mem));
 7484   predicate(!needs_acquiring_load(n));
 7485 
 7486   ins_cost(4 * INSN_COST);
 7487   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7488 
 7489   ins_encode(aarch64_enc_ldrw(dst, mem));
 7490 
 7491   ins_pipe(iload_reg_mem);
 7492 %}
 7493 
 7494 // Load Float
 7495 instruct loadF(vRegF dst, memory4 mem)
 7496 %{
 7497   match(Set dst (LoadF mem));
 7498   predicate(!needs_acquiring_load(n));
 7499 
 7500   ins_cost(4 * INSN_COST);
 7501   format %{ "ldrs  $dst, $mem\t# float" %}
 7502 
 7503   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7504 
 7505   ins_pipe(pipe_class_memory);
 7506 %}
 7507 
 7508 // Load Double
 7509 instruct loadD(vRegD dst, memory8 mem)
 7510 %{
 7511   match(Set dst (LoadD mem));
 7512   predicate(!needs_acquiring_load(n));
 7513 
 7514   ins_cost(4 * INSN_COST);
 7515   format %{ "ldrd  $dst, $mem\t# double" %}
 7516 
 7517   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7518 
 7519   ins_pipe(pipe_class_memory);
 7520 %}
 7521 
 7522 
 7523 // Load Int Constant
 7524 instruct loadConI(iRegINoSp dst, immI src)
 7525 %{
 7526   match(Set dst src);
 7527 
 7528   ins_cost(INSN_COST);
 7529   format %{ "mov $dst, $src\t# int" %}
 7530 
 7531   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7532 
 7533   ins_pipe(ialu_imm);
 7534 %}
 7535 
 7536 // Load Long Constant
 7537 instruct loadConL(iRegLNoSp dst, immL src)
 7538 %{
 7539   match(Set dst src);
 7540 
 7541   ins_cost(INSN_COST);
 7542   format %{ "mov $dst, $src\t# long" %}
 7543 
 7544   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7545 
 7546   ins_pipe(ialu_imm);
 7547 %}
 7548 
 7549 // Load Pointer Constant
 7550 
 7551 instruct loadConP(iRegPNoSp dst, immP con)
 7552 %{
 7553   match(Set dst con);
 7554 
 7555   ins_cost(INSN_COST * 4);
 7556   format %{
 7557     "mov  $dst, $con\t# ptr\n\t"
 7558   %}
 7559 
 7560   ins_encode(aarch64_enc_mov_p(dst, con));
 7561 
 7562   ins_pipe(ialu_imm);
 7563 %}
 7564 
 7565 // Load Null Pointer Constant
 7566 
 7567 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7568 %{
 7569   match(Set dst con);
 7570 
 7571   ins_cost(INSN_COST);
 7572   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7573 
 7574   ins_encode(aarch64_enc_mov_p0(dst, con));
 7575 
 7576   ins_pipe(ialu_imm);
 7577 %}
 7578 
 7579 // Load Pointer Constant One
 7580 
 7581 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7582 %{
 7583   match(Set dst con);
 7584 
 7585   ins_cost(INSN_COST);
 7586   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7587 
 7588   ins_encode(aarch64_enc_mov_p1(dst, con));
 7589 
 7590   ins_pipe(ialu_imm);
 7591 %}
 7592 
 7593 // Load Byte Map Base Constant
 7594 
 7595 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7596 %{
 7597   match(Set dst con);
 7598 
 7599   ins_cost(INSN_COST);
 7600   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7601 
 7602   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7603 
 7604   ins_pipe(ialu_imm);
 7605 %}
 7606 
 7607 // Load Narrow Pointer Constant
 7608 
 7609 instruct loadConN(iRegNNoSp dst, immN con)
 7610 %{
 7611   match(Set dst con);
 7612 
 7613   ins_cost(INSN_COST * 4);
 7614   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7615 
 7616   ins_encode(aarch64_enc_mov_n(dst, con));
 7617 
 7618   ins_pipe(ialu_imm);
 7619 %}
 7620 
 7621 // Load Narrow Null Pointer Constant
 7622 
 7623 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7624 %{
 7625   match(Set dst con);
 7626 
 7627   ins_cost(INSN_COST);
 7628   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7629 
 7630   ins_encode(aarch64_enc_mov_n0(dst, con));
 7631 
 7632   ins_pipe(ialu_imm);
 7633 %}
 7634 
 7635 // Load Narrow Klass Constant
 7636 
 7637 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7638 %{
 7639   match(Set dst con);
 7640 
 7641   ins_cost(INSN_COST);
 7642   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7643 
 7644   ins_encode(aarch64_enc_mov_nk(dst, con));
 7645 
 7646   ins_pipe(ialu_imm);
 7647 %}
 7648 
 7649 // Load Packed Float Constant
 7650 
 7651 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7652   match(Set dst con);
 7653   ins_cost(INSN_COST * 4);
 7654   format %{ "fmovs  $dst, $con"%}
 7655   ins_encode %{
 7656     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7657   %}
 7658 
 7659   ins_pipe(fp_imm_s);
 7660 %}
 7661 
 7662 // Load Float Constant
 7663 
 7664 instruct loadConF(vRegF dst, immF con) %{
 7665   match(Set dst con);
 7666 
 7667   ins_cost(INSN_COST * 4);
 7668 
 7669   format %{
 7670     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7671   %}
 7672 
 7673   ins_encode %{
 7674     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7675   %}
 7676 
 7677   ins_pipe(fp_load_constant_s);
 7678 %}
 7679 
 7680 // Load Packed Double Constant
 7681 
 7682 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7683   match(Set dst con);
 7684   ins_cost(INSN_COST);
 7685   format %{ "fmovd  $dst, $con"%}
 7686   ins_encode %{
 7687     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7688   %}
 7689 
 7690   ins_pipe(fp_imm_d);
 7691 %}
 7692 
 7693 // Load Double Constant
 7694 
 7695 instruct loadConD(vRegD dst, immD con) %{
 7696   match(Set dst con);
 7697 
 7698   ins_cost(INSN_COST * 5);
 7699   format %{
 7700     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7701   %}
 7702 
 7703   ins_encode %{
 7704     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7705   %}
 7706 
 7707   ins_pipe(fp_load_constant_d);
 7708 %}
 7709 
 7710 // Store Instructions
 7711 
 7712 // Store CMS card-mark Immediate
 7713 instruct storeimmCM0(immI0 zero, memory1 mem)
 7714 %{
 7715   match(Set mem (StoreCM mem zero));
 7716 
 7717   ins_cost(INSN_COST);
 7718   format %{ "storestore (elided)\n\t"
 7719             "strb zr, $mem\t# byte" %}
 7720 
 7721   ins_encode(aarch64_enc_strb0(mem));
 7722 
 7723   ins_pipe(istore_mem);
 7724 %}
 7725 
 7726 // Store CMS card-mark Immediate with intervening StoreStore
 7727 // needed when using CMS with no conditional card marking
 7728 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7729 %{
 7730   match(Set mem (StoreCM mem zero));
 7731 
 7732   ins_cost(INSN_COST * 2);
 7733   format %{ "storestore\n\t"
 7734             "dmb ishst"
 7735             "\n\tstrb zr, $mem\t# byte" %}
 7736 
 7737   ins_encode(aarch64_enc_strb0_ordered(mem));
 7738 
 7739   ins_pipe(istore_mem);
 7740 %}
 7741 
 7742 // Store Byte
 7743 instruct storeB(iRegIorL2I src, memory1 mem)
 7744 %{
 7745   match(Set mem (StoreB mem src));
 7746   predicate(!needs_releasing_store(n));
 7747 
 7748   ins_cost(INSN_COST);
 7749   format %{ "strb  $src, $mem\t# byte" %}
 7750 
 7751   ins_encode(aarch64_enc_strb(src, mem));
 7752 
 7753   ins_pipe(istore_reg_mem);
 7754 %}
 7755 
 7756 
 7757 instruct storeimmB0(immI0 zero, memory1 mem)
 7758 %{
 7759   match(Set mem (StoreB mem zero));
 7760   predicate(!needs_releasing_store(n));
 7761 
 7762   ins_cost(INSN_COST);
 7763   format %{ "strb rscractch2, $mem\t# byte" %}
 7764 
 7765   ins_encode(aarch64_enc_strb0(mem));
 7766 
 7767   ins_pipe(istore_mem);
 7768 %}
 7769 
 7770 // Store Char/Short
 7771 instruct storeC(iRegIorL2I src, memory2 mem)
 7772 %{
 7773   match(Set mem (StoreC mem src));
 7774   predicate(!needs_releasing_store(n));
 7775 
 7776   ins_cost(INSN_COST);
 7777   format %{ "strh  $src, $mem\t# short" %}
 7778 
 7779   ins_encode(aarch64_enc_strh(src, mem));
 7780 
 7781   ins_pipe(istore_reg_mem);
 7782 %}
 7783 
 7784 instruct storeimmC0(immI0 zero, memory2 mem)
 7785 %{
 7786   match(Set mem (StoreC mem zero));
 7787   predicate(!needs_releasing_store(n));
 7788 
 7789   ins_cost(INSN_COST);
 7790   format %{ "strh  zr, $mem\t# short" %}
 7791 
 7792   ins_encode(aarch64_enc_strh0(mem));
 7793 
 7794   ins_pipe(istore_mem);
 7795 %}
 7796 
 7797 // Store Integer
 7798 
 7799 instruct storeI(iRegIorL2I src, memory4 mem)
 7800 %{
 7801   match(Set mem(StoreI mem src));
 7802   predicate(!needs_releasing_store(n));
 7803 
 7804   ins_cost(INSN_COST);
 7805   format %{ "strw  $src, $mem\t# int" %}
 7806 
 7807   ins_encode(aarch64_enc_strw(src, mem));
 7808 
 7809   ins_pipe(istore_reg_mem);
 7810 %}
 7811 
 7812 instruct storeimmI0(immI0 zero, memory4 mem)
 7813 %{
 7814   match(Set mem(StoreI mem zero));
 7815   predicate(!needs_releasing_store(n));
 7816 
 7817   ins_cost(INSN_COST);
 7818   format %{ "strw  zr, $mem\t# int" %}
 7819 
 7820   ins_encode(aarch64_enc_strw0(mem));
 7821 
 7822   ins_pipe(istore_mem);
 7823 %}
 7824 
 7825 // Store Long (64 bit signed)
 7826 instruct storeL(iRegL src, memory8 mem)
 7827 %{
 7828   match(Set mem (StoreL mem src));
 7829   predicate(!needs_releasing_store(n));
 7830 
 7831   ins_cost(INSN_COST);
 7832   format %{ "str  $src, $mem\t# int" %}
 7833 
 7834   ins_encode(aarch64_enc_str(src, mem));
 7835 
 7836   ins_pipe(istore_reg_mem);
 7837 %}
 7838 
 7839 // Store Long (64 bit signed)
 7840 instruct storeimmL0(immL0 zero, memory8 mem)
 7841 %{
 7842   match(Set mem (StoreL mem zero));
 7843   predicate(!needs_releasing_store(n));
 7844 
 7845   ins_cost(INSN_COST);
 7846   format %{ "str  zr, $mem\t# int" %}
 7847 
 7848   ins_encode(aarch64_enc_str0(mem));
 7849 
 7850   ins_pipe(istore_mem);
 7851 %}
 7852 
 7853 // Store Pointer
 7854 instruct storeP(iRegP src, memory8 mem)
 7855 %{
 7856   match(Set mem (StoreP mem src));
 7857   predicate(!needs_releasing_store(n));
 7858 
 7859   ins_cost(INSN_COST);
 7860   format %{ "str  $src, $mem\t# ptr" %}
 7861 
 7862   ins_encode(aarch64_enc_str(src, mem));
 7863 
 7864   ins_pipe(istore_reg_mem);
 7865 %}
 7866 
 7867 // Store Pointer
 7868 instruct storeimmP0(immP0 zero, memory8 mem)
 7869 %{
 7870   match(Set mem (StoreP mem zero));
 7871   predicate(!needs_releasing_store(n));
 7872 
 7873   ins_cost(INSN_COST);
 7874   format %{ "str zr, $mem\t# ptr" %}
 7875 
 7876   ins_encode(aarch64_enc_str0(mem));
 7877 
 7878   ins_pipe(istore_mem);
 7879 %}
 7880 
 7881 // Store Compressed Pointer
 7882 instruct storeN(iRegN src, memory4 mem)
 7883 %{
 7884   match(Set mem (StoreN mem src));
 7885   predicate(!needs_releasing_store(n));
 7886 
 7887   ins_cost(INSN_COST);
 7888   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7889 
 7890   ins_encode(aarch64_enc_strw(src, mem));
 7891 
 7892   ins_pipe(istore_reg_mem);
 7893 %}
 7894 
 7895 instruct storeImmN0(immN0 zero, memory4 mem)
 7896 %{
 7897   match(Set mem (StoreN mem zero));
 7898   predicate(!needs_releasing_store(n));
 7899 
 7900   ins_cost(INSN_COST);
 7901   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7902 
 7903   ins_encode(aarch64_enc_strw0(mem));
 7904 
 7905   ins_pipe(istore_mem);
 7906 %}
 7907 
 7908 // Store Float
 7909 instruct storeF(vRegF src, memory4 mem)
 7910 %{
 7911   match(Set mem (StoreF mem src));
 7912   predicate(!needs_releasing_store(n));
 7913 
 7914   ins_cost(INSN_COST);
 7915   format %{ "strs  $src, $mem\t# float" %}
 7916 
 7917   ins_encode( aarch64_enc_strs(src, mem) );
 7918 
 7919   ins_pipe(pipe_class_memory);
 7920 %}
 7921 
 7922 // TODO
 7923 // implement storeImmF0 and storeFImmPacked
 7924 
 7925 // Store Double
 7926 instruct storeD(vRegD src, memory8 mem)
 7927 %{
 7928   match(Set mem (StoreD mem src));
 7929   predicate(!needs_releasing_store(n));
 7930 
 7931   ins_cost(INSN_COST);
 7932   format %{ "strd  $src, $mem\t# double" %}
 7933 
 7934   ins_encode( aarch64_enc_strd(src, mem) );
 7935 
 7936   ins_pipe(pipe_class_memory);
 7937 %}
 7938 
 7939 // Store Compressed Klass Pointer
 7940 instruct storeNKlass(iRegN src, memory4 mem)
 7941 %{
 7942   predicate(!needs_releasing_store(n));
 7943   match(Set mem (StoreNKlass mem src));
 7944 
 7945   ins_cost(INSN_COST);
 7946   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7947 
 7948   ins_encode(aarch64_enc_strw(src, mem));
 7949 
 7950   ins_pipe(istore_reg_mem);
 7951 %}
 7952 
 7953 // TODO
 7954 // implement storeImmD0 and storeDImmPacked
 7955 
 7956 // prefetch instructions
 7957 // Must be safe to execute with invalid address (cannot fault).
 7958 
 7959 instruct prefetchalloc( memory8 mem ) %{
 7960   match(PrefetchAllocation mem);
 7961 
 7962   ins_cost(INSN_COST);
 7963   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7964 
 7965   ins_encode( aarch64_enc_prefetchw(mem) );
 7966 
 7967   ins_pipe(iload_prefetch);
 7968 %}
 7969 
 7970 //  ---------------- volatile loads and stores ----------------
 7971 
 7972 // Load Byte (8 bit signed)
 7973 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7974 %{
 7975   match(Set dst (LoadB mem));
 7976 
 7977   ins_cost(VOLATILE_REF_COST);
 7978   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7979 
 7980   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7981 
 7982   ins_pipe(pipe_serial);
 7983 %}
 7984 
 7985 // Load Byte (8 bit signed) into long
 7986 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7987 %{
 7988   match(Set dst (ConvI2L (LoadB mem)));
 7989 
 7990   ins_cost(VOLATILE_REF_COST);
 7991   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7992 
 7993   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7994 
 7995   ins_pipe(pipe_serial);
 7996 %}
 7997 
 7998 // Load Byte (8 bit unsigned)
 7999 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8000 %{
 8001   match(Set dst (LoadUB mem));
 8002 
 8003   ins_cost(VOLATILE_REF_COST);
 8004   format %{ "ldarb  $dst, $mem\t# byte" %}
 8005 
 8006   ins_encode(aarch64_enc_ldarb(dst, mem));
 8007 
 8008   ins_pipe(pipe_serial);
 8009 %}
 8010 
 8011 // Load Byte (8 bit unsigned) into long
 8012 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8013 %{
 8014   match(Set dst (ConvI2L (LoadUB mem)));
 8015 
 8016   ins_cost(VOLATILE_REF_COST);
 8017   format %{ "ldarb  $dst, $mem\t# byte" %}
 8018 
 8019   ins_encode(aarch64_enc_ldarb(dst, mem));
 8020 
 8021   ins_pipe(pipe_serial);
 8022 %}
 8023 
 8024 // Load Short (16 bit signed)
 8025 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8026 %{
 8027   match(Set dst (LoadS mem));
 8028 
 8029   ins_cost(VOLATILE_REF_COST);
 8030   format %{ "ldarshw  $dst, $mem\t# short" %}
 8031 
 8032   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8033 
 8034   ins_pipe(pipe_serial);
 8035 %}
 8036 
 8037 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8038 %{
 8039   match(Set dst (LoadUS mem));
 8040 
 8041   ins_cost(VOLATILE_REF_COST);
 8042   format %{ "ldarhw  $dst, $mem\t# short" %}
 8043 
 8044   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8045 
 8046   ins_pipe(pipe_serial);
 8047 %}
 8048 
 8049 // Load Short/Char (16 bit unsigned) into long
 8050 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8051 %{
 8052   match(Set dst (ConvI2L (LoadUS mem)));
 8053 
 8054   ins_cost(VOLATILE_REF_COST);
 8055   format %{ "ldarh  $dst, $mem\t# short" %}
 8056 
 8057   ins_encode(aarch64_enc_ldarh(dst, mem));
 8058 
 8059   ins_pipe(pipe_serial);
 8060 %}
 8061 
 8062 // Load Short/Char (16 bit signed) into long
 8063 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8064 %{
 8065   match(Set dst (ConvI2L (LoadS mem)));
 8066 
 8067   ins_cost(VOLATILE_REF_COST);
 8068   format %{ "ldarh  $dst, $mem\t# short" %}
 8069 
 8070   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8071 
 8072   ins_pipe(pipe_serial);
 8073 %}
 8074 
 8075 // Load Integer (32 bit signed)
 8076 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8077 %{
 8078   match(Set dst (LoadI mem));
 8079 
 8080   ins_cost(VOLATILE_REF_COST);
 8081   format %{ "ldarw  $dst, $mem\t# int" %}
 8082 
 8083   ins_encode(aarch64_enc_ldarw(dst, mem));
 8084 
 8085   ins_pipe(pipe_serial);
 8086 %}
 8087 
 8088 // Load Integer (32 bit unsigned) into long
 8089 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8090 %{
 8091   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8092 
 8093   ins_cost(VOLATILE_REF_COST);
 8094   format %{ "ldarw  $dst, $mem\t# int" %}
 8095 
 8096   ins_encode(aarch64_enc_ldarw(dst, mem));
 8097 
 8098   ins_pipe(pipe_serial);
 8099 %}
 8100 
 8101 // Load Long (64 bit signed)
 8102 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8103 %{
 8104   match(Set dst (LoadL mem));
 8105 
 8106   ins_cost(VOLATILE_REF_COST);
 8107   format %{ "ldar  $dst, $mem\t# int" %}
 8108 
 8109   ins_encode(aarch64_enc_ldar(dst, mem));
 8110 
 8111   ins_pipe(pipe_serial);
 8112 %}
 8113 
 8114 // Load Pointer
 8115 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8116 %{
 8117   match(Set dst (LoadP mem));
 8118   predicate(n->as_Load()->barrier_data() == 0);
 8119 
 8120   ins_cost(VOLATILE_REF_COST);
 8121   format %{ "ldar  $dst, $mem\t# ptr" %}
 8122 
 8123   ins_encode(aarch64_enc_ldar(dst, mem));
 8124 
 8125   ins_pipe(pipe_serial);
 8126 %}
 8127 
 8128 // Load Compressed Pointer
 8129 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8130 %{
 8131   match(Set dst (LoadN mem));
 8132 
 8133   ins_cost(VOLATILE_REF_COST);
 8134   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8135 
 8136   ins_encode(aarch64_enc_ldarw(dst, mem));
 8137 
 8138   ins_pipe(pipe_serial);
 8139 %}
 8140 
 8141 // Load Float
 8142 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8143 %{
 8144   match(Set dst (LoadF mem));
 8145 
 8146   ins_cost(VOLATILE_REF_COST);
 8147   format %{ "ldars  $dst, $mem\t# float" %}
 8148 
 8149   ins_encode( aarch64_enc_fldars(dst, mem) );
 8150 
 8151   ins_pipe(pipe_serial);
 8152 %}
 8153 
 8154 // Load Double
 8155 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8156 %{
 8157   match(Set dst (LoadD mem));
 8158 
 8159   ins_cost(VOLATILE_REF_COST);
 8160   format %{ "ldard  $dst, $mem\t# double" %}
 8161 
 8162   ins_encode( aarch64_enc_fldard(dst, mem) );
 8163 
 8164   ins_pipe(pipe_serial);
 8165 %}
 8166 
 8167 // Store Byte
 8168 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8169 %{
 8170   match(Set mem (StoreB mem src));
 8171 
 8172   ins_cost(VOLATILE_REF_COST);
 8173   format %{ "stlrb  $src, $mem\t# byte" %}
 8174 
 8175   ins_encode(aarch64_enc_stlrb(src, mem));
 8176 
 8177   ins_pipe(pipe_class_memory);
 8178 %}
 8179 
 8180 // Store Char/Short
 8181 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8182 %{
 8183   match(Set mem (StoreC mem src));
 8184 
 8185   ins_cost(VOLATILE_REF_COST);
 8186   format %{ "stlrh  $src, $mem\t# short" %}
 8187 
 8188   ins_encode(aarch64_enc_stlrh(src, mem));
 8189 
 8190   ins_pipe(pipe_class_memory);
 8191 %}
 8192 
 8193 // Store Integer
 8194 
 8195 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8196 %{
 8197   match(Set mem(StoreI mem src));
 8198 
 8199   ins_cost(VOLATILE_REF_COST);
 8200   format %{ "stlrw  $src, $mem\t# int" %}
 8201 
 8202   ins_encode(aarch64_enc_stlrw(src, mem));
 8203 
 8204   ins_pipe(pipe_class_memory);
 8205 %}
 8206 
 8207 // Store Long (64 bit signed)
 8208 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8209 %{
 8210   match(Set mem (StoreL mem src));
 8211 
 8212   ins_cost(VOLATILE_REF_COST);
 8213   format %{ "stlr  $src, $mem\t# int" %}
 8214 
 8215   ins_encode(aarch64_enc_stlr(src, mem));
 8216 
 8217   ins_pipe(pipe_class_memory);
 8218 %}
 8219 
 8220 // Store Pointer
 8221 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8222 %{
 8223   match(Set mem (StoreP mem src));
 8224 
 8225   ins_cost(VOLATILE_REF_COST);
 8226   format %{ "stlr  $src, $mem\t# ptr" %}
 8227 
 8228   ins_encode(aarch64_enc_stlr(src, mem));
 8229 
 8230   ins_pipe(pipe_class_memory);
 8231 %}
 8232 
 8233 // Store Compressed Pointer
 8234 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8235 %{
 8236   match(Set mem (StoreN mem src));
 8237 
 8238   ins_cost(VOLATILE_REF_COST);
 8239   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8240 
 8241   ins_encode(aarch64_enc_stlrw(src, mem));
 8242 
 8243   ins_pipe(pipe_class_memory);
 8244 %}
 8245 
 8246 // Store Float
 8247 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8248 %{
 8249   match(Set mem (StoreF mem src));
 8250 
 8251   ins_cost(VOLATILE_REF_COST);
 8252   format %{ "stlrs  $src, $mem\t# float" %}
 8253 
 8254   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8255 
 8256   ins_pipe(pipe_class_memory);
 8257 %}
 8258 
 8259 // TODO
 8260 // implement storeImmF0 and storeFImmPacked
 8261 
 8262 // Store Double
 8263 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8264 %{
 8265   match(Set mem (StoreD mem src));
 8266 
 8267   ins_cost(VOLATILE_REF_COST);
 8268   format %{ "stlrd  $src, $mem\t# double" %}
 8269 
 8270   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8271 
 8272   ins_pipe(pipe_class_memory);
 8273 %}
 8274 
 8275 //  ---------------- end of volatile loads and stores ----------------
 8276 
 8277 instruct cacheWB(indirect addr)
 8278 %{
 8279   predicate(VM_Version::supports_data_cache_line_flush());
 8280   match(CacheWB addr);
 8281 
 8282   ins_cost(100);
 8283   format %{"cache wb $addr" %}
 8284   ins_encode %{
 8285     assert($addr->index_position() < 0, "should be");
 8286     assert($addr$$disp == 0, "should be");
 8287     __ cache_wb(Address($addr$$base$$Register, 0));
 8288   %}
 8289   ins_pipe(pipe_slow); // XXX
 8290 %}
 8291 
 8292 instruct cacheWBPreSync()
 8293 %{
 8294   predicate(VM_Version::supports_data_cache_line_flush());
 8295   match(CacheWBPreSync);
 8296 
 8297   ins_cost(100);
 8298   format %{"cache wb presync" %}
 8299   ins_encode %{
 8300     __ cache_wbsync(true);
 8301   %}
 8302   ins_pipe(pipe_slow); // XXX
 8303 %}
 8304 
 8305 instruct cacheWBPostSync()
 8306 %{
 8307   predicate(VM_Version::supports_data_cache_line_flush());
 8308   match(CacheWBPostSync);
 8309 
 8310   ins_cost(100);
 8311   format %{"cache wb postsync" %}
 8312   ins_encode %{
 8313     __ cache_wbsync(false);
 8314   %}
 8315   ins_pipe(pipe_slow); // XXX
 8316 %}
 8317 
 8318 // ============================================================================
 8319 // BSWAP Instructions
 8320 
 8321 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8322   match(Set dst (ReverseBytesI src));
 8323 
 8324   ins_cost(INSN_COST);
 8325   format %{ "revw  $dst, $src" %}
 8326 
 8327   ins_encode %{
 8328     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8329   %}
 8330 
 8331   ins_pipe(ialu_reg);
 8332 %}
 8333 
 8334 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8335   match(Set dst (ReverseBytesL src));
 8336 
 8337   ins_cost(INSN_COST);
 8338   format %{ "rev  $dst, $src" %}
 8339 
 8340   ins_encode %{
 8341     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8342   %}
 8343 
 8344   ins_pipe(ialu_reg);
 8345 %}
 8346 
 8347 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8348   match(Set dst (ReverseBytesUS src));
 8349 
 8350   ins_cost(INSN_COST);
 8351   format %{ "rev16w  $dst, $src" %}
 8352 
 8353   ins_encode %{
 8354     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8355   %}
 8356 
 8357   ins_pipe(ialu_reg);
 8358 %}
 8359 
 8360 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8361   match(Set dst (ReverseBytesS src));
 8362 
 8363   ins_cost(INSN_COST);
 8364   format %{ "rev16w  $dst, $src\n\t"
 8365             "sbfmw $dst, $dst, #0, #15" %}
 8366 
 8367   ins_encode %{
 8368     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8369     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8370   %}
 8371 
 8372   ins_pipe(ialu_reg);
 8373 %}
 8374 
 8375 // ============================================================================
 8376 // Zero Count Instructions
 8377 
 8378 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8379   match(Set dst (CountLeadingZerosI src));
 8380 
 8381   ins_cost(INSN_COST);
 8382   format %{ "clzw  $dst, $src" %}
 8383   ins_encode %{
 8384     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8385   %}
 8386 
 8387   ins_pipe(ialu_reg);
 8388 %}
 8389 
 8390 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8391   match(Set dst (CountLeadingZerosL src));
 8392 
 8393   ins_cost(INSN_COST);
 8394   format %{ "clz   $dst, $src" %}
 8395   ins_encode %{
 8396     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8397   %}
 8398 
 8399   ins_pipe(ialu_reg);
 8400 %}
 8401 
 8402 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8403   match(Set dst (CountTrailingZerosI src));
 8404 
 8405   ins_cost(INSN_COST * 2);
 8406   format %{ "rbitw  $dst, $src\n\t"
 8407             "clzw   $dst, $dst" %}
 8408   ins_encode %{
 8409     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8410     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8411   %}
 8412 
 8413   ins_pipe(ialu_reg);
 8414 %}
 8415 
 8416 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8417   match(Set dst (CountTrailingZerosL src));
 8418 
 8419   ins_cost(INSN_COST * 2);
 8420   format %{ "rbit   $dst, $src\n\t"
 8421             "clz    $dst, $dst" %}
 8422   ins_encode %{
 8423     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8424     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8425   %}
 8426 
 8427   ins_pipe(ialu_reg);
 8428 %}
 8429 
 8430 //---------- Population Count Instructions -------------------------------------
 8431 //
 8432 
 8433 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8434   predicate(UsePopCountInstruction);
 8435   match(Set dst (PopCountI src));
 8436   effect(TEMP tmp);
 8437   ins_cost(INSN_COST * 13);
 8438 
 8439   format %{ "movw   $src, $src\n\t"
 8440             "mov    $tmp, $src\t# vector (1D)\n\t"
 8441             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8442             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8443             "mov    $dst, $tmp\t# vector (1D)" %}
 8444   ins_encode %{
 8445     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8446     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8447     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8448     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8449     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8450   %}
 8451 
 8452   ins_pipe(pipe_class_default);
 8453 %}
 8454 
 8455 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8456   predicate(UsePopCountInstruction);
 8457   match(Set dst (PopCountI (LoadI mem)));
 8458   effect(TEMP tmp);
 8459   ins_cost(INSN_COST * 13);
 8460 
 8461   format %{ "ldrs   $tmp, $mem\n\t"
 8462             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8463             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8464             "mov    $dst, $tmp\t# vector (1D)" %}
 8465   ins_encode %{
 8466     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8467     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8468               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8469     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8470     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8471     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8472   %}
 8473 
 8474   ins_pipe(pipe_class_default);
 8475 %}
 8476 
 8477 // Note: Long.bitCount(long) returns an int.
 8478 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8479   predicate(UsePopCountInstruction);
 8480   match(Set dst (PopCountL src));
 8481   effect(TEMP tmp);
 8482   ins_cost(INSN_COST * 13);
 8483 
 8484   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8485             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8486             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8487             "mov    $dst, $tmp\t# vector (1D)" %}
 8488   ins_encode %{
 8489     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8490     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8491     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8492     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8493   %}
 8494 
 8495   ins_pipe(pipe_class_default);
 8496 %}
 8497 
 8498 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8499   predicate(UsePopCountInstruction);
 8500   match(Set dst (PopCountL (LoadL mem)));
 8501   effect(TEMP tmp);
 8502   ins_cost(INSN_COST * 13);
 8503 
 8504   format %{ "ldrd   $tmp, $mem\n\t"
 8505             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8506             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8507             "mov    $dst, $tmp\t# vector (1D)" %}
 8508   ins_encode %{
 8509     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8510     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8511               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8512     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8513     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8514     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8515   %}
 8516 
 8517   ins_pipe(pipe_class_default);
 8518 %}
 8519 
 8520 // ============================================================================
 8521 // MemBar Instruction
 8522 
 8523 instruct load_fence() %{
 8524   match(LoadFence);
 8525   ins_cost(VOLATILE_REF_COST);
 8526 
 8527   format %{ "load_fence" %}
 8528 
 8529   ins_encode %{
 8530     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8531   %}
 8532   ins_pipe(pipe_serial);
 8533 %}
 8534 
 8535 instruct unnecessary_membar_acquire() %{
 8536   predicate(unnecessary_acquire(n));
 8537   match(MemBarAcquire);
 8538   ins_cost(0);
 8539 
 8540   format %{ "membar_acquire (elided)" %}
 8541 
 8542   ins_encode %{
 8543     __ block_comment("membar_acquire (elided)");
 8544   %}
 8545 
 8546   ins_pipe(pipe_class_empty);
 8547 %}
 8548 
 8549 instruct membar_acquire() %{
 8550   match(MemBarAcquire);
 8551   ins_cost(VOLATILE_REF_COST);
 8552 
 8553   format %{ "membar_acquire\n\t"
 8554             "dmb ish" %}
 8555 
 8556   ins_encode %{
 8557     __ block_comment("membar_acquire");
 8558     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8559   %}
 8560 
 8561   ins_pipe(pipe_serial);
 8562 %}
 8563 
 8564 
 8565 instruct membar_acquire_lock() %{
 8566   match(MemBarAcquireLock);
 8567   ins_cost(VOLATILE_REF_COST);
 8568 
 8569   format %{ "membar_acquire_lock (elided)" %}
 8570 
 8571   ins_encode %{
 8572     __ block_comment("membar_acquire_lock (elided)");
 8573   %}
 8574 
 8575   ins_pipe(pipe_serial);
 8576 %}
 8577 
 8578 instruct store_fence() %{
 8579   match(StoreFence);
 8580   ins_cost(VOLATILE_REF_COST);
 8581 
 8582   format %{ "store_fence" %}
 8583 
 8584   ins_encode %{
 8585     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8586   %}
 8587   ins_pipe(pipe_serial);
 8588 %}
 8589 
 8590 instruct unnecessary_membar_release() %{
 8591   predicate(unnecessary_release(n));
 8592   match(MemBarRelease);
 8593   ins_cost(0);
 8594 
 8595   format %{ "membar_release (elided)" %}
 8596 
 8597   ins_encode %{
 8598     __ block_comment("membar_release (elided)");
 8599   %}
 8600   ins_pipe(pipe_serial);
 8601 %}
 8602 
 8603 instruct membar_release() %{
 8604   match(MemBarRelease);
 8605   ins_cost(VOLATILE_REF_COST);
 8606 
 8607   format %{ "membar_release\n\t"
 8608             "dmb ish" %}
 8609 
 8610   ins_encode %{
 8611     __ block_comment("membar_release");
 8612     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8613   %}
 8614   ins_pipe(pipe_serial);
 8615 %}
 8616 
 8617 instruct membar_storestore() %{
 8618   match(MemBarStoreStore);
 8619   ins_cost(VOLATILE_REF_COST);
 8620 
 8621   format %{ "MEMBAR-store-store" %}
 8622 
 8623   ins_encode %{
 8624     __ membar(Assembler::StoreStore);
 8625   %}
 8626   ins_pipe(pipe_serial);
 8627 %}
 8628 
 8629 instruct membar_release_lock() %{
 8630   match(MemBarReleaseLock);
 8631   ins_cost(VOLATILE_REF_COST);
 8632 
 8633   format %{ "membar_release_lock (elided)" %}
 8634 
 8635   ins_encode %{
 8636     __ block_comment("membar_release_lock (elided)");
 8637   %}
 8638 
 8639   ins_pipe(pipe_serial);
 8640 %}
 8641 
 8642 instruct unnecessary_membar_volatile() %{
 8643   predicate(unnecessary_volatile(n));
 8644   match(MemBarVolatile);
 8645   ins_cost(0);
 8646 
 8647   format %{ "membar_volatile (elided)" %}
 8648 
 8649   ins_encode %{
 8650     __ block_comment("membar_volatile (elided)");
 8651   %}
 8652 
 8653   ins_pipe(pipe_serial);
 8654 %}
 8655 
 8656 instruct membar_volatile() %{
 8657   match(MemBarVolatile);
 8658   ins_cost(VOLATILE_REF_COST*100);
 8659 
 8660   format %{ "membar_volatile\n\t"
 8661              "dmb ish"%}
 8662 
 8663   ins_encode %{
 8664     __ block_comment("membar_volatile");
 8665     __ membar(Assembler::StoreLoad);
 8666   %}
 8667 
 8668   ins_pipe(pipe_serial);
 8669 %}
 8670 
 8671 // ============================================================================
 8672 // Cast/Convert Instructions
 8673 
 8674 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8675   match(Set dst (CastX2P src));
 8676 
 8677   ins_cost(INSN_COST);
 8678   format %{ "mov $dst, $src\t# long -> ptr" %}
 8679 
 8680   ins_encode %{
 8681     if ($dst$$reg != $src$$reg) {
 8682       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8683     }
 8684   %}
 8685 
 8686   ins_pipe(ialu_reg);
 8687 %}
 8688 
 8689 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8690   match(Set dst (CastP2X src));
 8691 
 8692   ins_cost(INSN_COST);
 8693   format %{ "mov $dst, $src\t# ptr -> long" %}
 8694 
 8695   ins_encode %{
 8696     if ($dst$$reg != $src$$reg) {
 8697       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8698     }
 8699   %}
 8700 
 8701   ins_pipe(ialu_reg);
 8702 %}
 8703 
 8704 // Convert oop into int for vectors alignment masking
 8705 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8706   match(Set dst (ConvL2I (CastP2X src)));
 8707 
 8708   ins_cost(INSN_COST);
 8709   format %{ "movw $dst, $src\t# ptr -> int" %}
 8710   ins_encode %{
 8711     __ movw($dst$$Register, $src$$Register);
 8712   %}
 8713 
 8714   ins_pipe(ialu_reg);
 8715 %}
 8716 
 8717 // Convert compressed oop into int for vectors alignment masking
 8718 // in case of 32bit oops (heap < 4Gb).
 8719 instruct convN2I(iRegINoSp dst, iRegN src)
 8720 %{
 8721   predicate(CompressedOops::shift() == 0);
 8722   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8723 
 8724   ins_cost(INSN_COST);
 8725   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8726   ins_encode %{
 8727     __ movw($dst$$Register, $src$$Register);
 8728   %}
 8729 
 8730   ins_pipe(ialu_reg);
 8731 %}
 8732 
 8733 
 8734 // Convert oop pointer into compressed form
 8735 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8736   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8737   match(Set dst (EncodeP src));
 8738   effect(KILL cr);
 8739   ins_cost(INSN_COST * 3);
 8740   format %{ "encode_heap_oop $dst, $src" %}
 8741   ins_encode %{
 8742     Register s = $src$$Register;
 8743     Register d = $dst$$Register;
 8744     __ encode_heap_oop(d, s);
 8745   %}
 8746   ins_pipe(ialu_reg);
 8747 %}
 8748 
 8749 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8750   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8751   match(Set dst (EncodeP src));
 8752   ins_cost(INSN_COST * 3);
 8753   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8754   ins_encode %{
 8755     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8756   %}
 8757   ins_pipe(ialu_reg);
 8758 %}
 8759 
 8760 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8761   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8762             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8763   match(Set dst (DecodeN src));
 8764   ins_cost(INSN_COST * 3);
 8765   format %{ "decode_heap_oop $dst, $src" %}
 8766   ins_encode %{
 8767     Register s = $src$$Register;
 8768     Register d = $dst$$Register;
 8769     __ decode_heap_oop(d, s);
 8770   %}
 8771   ins_pipe(ialu_reg);
 8772 %}
 8773 
 8774 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8775   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8776             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8777   match(Set dst (DecodeN src));
 8778   ins_cost(INSN_COST * 3);
 8779   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8780   ins_encode %{
 8781     Register s = $src$$Register;
 8782     Register d = $dst$$Register;
 8783     __ decode_heap_oop_not_null(d, s);
 8784   %}
 8785   ins_pipe(ialu_reg);
 8786 %}
 8787 
 8788 // n.b. AArch64 implementations of encode_klass_not_null and
 8789 // decode_klass_not_null do not modify the flags register so, unlike
 8790 // Intel, we don't kill CR as a side effect here
 8791 
 8792 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8793   match(Set dst (EncodePKlass src));
 8794 
 8795   ins_cost(INSN_COST * 3);
 8796   format %{ "encode_klass_not_null $dst,$src" %}
 8797 
 8798   ins_encode %{
 8799     Register src_reg = as_Register($src$$reg);
 8800     Register dst_reg = as_Register($dst$$reg);
 8801     __ encode_klass_not_null(dst_reg, src_reg);
 8802   %}
 8803 
 8804    ins_pipe(ialu_reg);
 8805 %}
 8806 
 8807 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8808   match(Set dst (DecodeNKlass src));
 8809 
 8810   ins_cost(INSN_COST * 3);
 8811   format %{ "decode_klass_not_null $dst,$src" %}
 8812 
 8813   ins_encode %{
 8814     Register src_reg = as_Register($src$$reg);
 8815     Register dst_reg = as_Register($dst$$reg);
 8816     if (dst_reg != src_reg) {
 8817       __ decode_klass_not_null(dst_reg, src_reg);
 8818     } else {
 8819       __ decode_klass_not_null(dst_reg);
 8820     }
 8821   %}
 8822 
 8823    ins_pipe(ialu_reg);
 8824 %}
 8825 
 8826 instruct checkCastPP(iRegPNoSp dst)
 8827 %{
 8828   match(Set dst (CheckCastPP dst));
 8829 
 8830   size(0);
 8831   format %{ "# checkcastPP of $dst" %}
 8832   ins_encode(/* empty encoding */);
 8833   ins_pipe(pipe_class_empty);
 8834 %}
 8835 
 8836 instruct castPP(iRegPNoSp dst)
 8837 %{
 8838   match(Set dst (CastPP dst));
 8839 
 8840   size(0);
 8841   format %{ "# castPP of $dst" %}
 8842   ins_encode(/* empty encoding */);
 8843   ins_pipe(pipe_class_empty);
 8844 %}
 8845 
 8846 instruct castII(iRegI dst)
 8847 %{
 8848   match(Set dst (CastII dst));
 8849 
 8850   size(0);
 8851   format %{ "# castII of $dst" %}
 8852   ins_encode(/* empty encoding */);
 8853   ins_cost(0);
 8854   ins_pipe(pipe_class_empty);
 8855 %}
 8856 
 8857 instruct castLL(iRegL dst)
 8858 %{
 8859   match(Set dst (CastLL dst));
 8860 
 8861   size(0);
 8862   format %{ "# castLL of $dst" %}
 8863   ins_encode(/* empty encoding */);
 8864   ins_cost(0);
 8865   ins_pipe(pipe_class_empty);
 8866 %}
 8867 
 8868 instruct castFF(vRegF dst)
 8869 %{
 8870   match(Set dst (CastFF dst));
 8871 
 8872   size(0);
 8873   format %{ "# castFF of $dst" %}
 8874   ins_encode(/* empty encoding */);
 8875   ins_cost(0);
 8876   ins_pipe(pipe_class_empty);
 8877 %}
 8878 
 8879 instruct castDD(vRegD dst)
 8880 %{
 8881   match(Set dst (CastDD dst));
 8882 
 8883   size(0);
 8884   format %{ "# castDD of $dst" %}
 8885   ins_encode(/* empty encoding */);
 8886   ins_cost(0);
 8887   ins_pipe(pipe_class_empty);
 8888 %}
 8889 
 8890 instruct castVVD(vecD dst)
 8891 %{
 8892   match(Set dst (CastVV dst));
 8893 
 8894   size(0);
 8895   format %{ "# castVV of $dst" %}
 8896   ins_encode(/* empty encoding */);
 8897   ins_cost(0);
 8898   ins_pipe(pipe_class_empty);
 8899 %}
 8900 
 8901 instruct castVVX(vecX dst)
 8902 %{
 8903   match(Set dst (CastVV dst));
 8904 
 8905   size(0);
 8906   format %{ "# castVV of $dst" %}
 8907   ins_encode(/* empty encoding */);
 8908   ins_cost(0);
 8909   ins_pipe(pipe_class_empty);
 8910 %}
 8911 
 8912 instruct castVV(vReg dst)
 8913 %{
 8914   match(Set dst (CastVV dst));
 8915 
 8916   size(0);
 8917   format %{ "# castVV of $dst" %}
 8918   ins_encode(/* empty encoding */);
 8919   ins_cost(0);
 8920   ins_pipe(pipe_class_empty);
 8921 %}
 8922 
 8923 instruct castVVMask(pRegGov dst)
 8924 %{
 8925   match(Set dst (CastVV dst));
 8926 
 8927   size(0);
 8928   format %{ "# castVV of $dst" %}
 8929   ins_encode(/* empty encoding */);
 8930   ins_cost(0);
 8931   ins_pipe(pipe_class_empty);
 8932 %}
 8933 
 8934 // ============================================================================
 8935 // Atomic operation instructions
 8936 //
 8937 // Intel and SPARC both implement Ideal Node LoadPLocked and
 8938 // Store{PIL}Conditional instructions using a normal load for the
 8939 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 8940 //
 8941 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 8942 // pair to lock object allocations from Eden space when not using
 8943 // TLABs.
 8944 //
 8945 // There does not appear to be a Load{IL}Locked Ideal Node and the
 8946 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 8947 // and to use StoreIConditional only for 32-bit and StoreLConditional
 8948 // only for 64-bit.
 8949 //
 8950 // We implement LoadPLocked and StorePLocked instructions using,
 8951 // respectively the AArch64 hw load-exclusive and store-conditional
 8952 // instructions. Whereas we must implement each of
 8953 // Store{IL}Conditional using a CAS which employs a pair of
 8954 // instructions comprising a load-exclusive followed by a
 8955 // store-conditional.
 8956 
 8957 
 8958 // Locked-load (linked load) of the current heap-top
 8959 // used when updating the eden heap top
 8960 // implemented using ldaxr on AArch64
 8961 
 8962 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 8963 %{
 8964   match(Set dst (LoadPLocked mem));
 8965 
 8966   ins_cost(VOLATILE_REF_COST);
 8967 
 8968   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 8969 
 8970   ins_encode(aarch64_enc_ldaxr(dst, mem));
 8971 
 8972   ins_pipe(pipe_serial);
 8973 %}
 8974 
 8975 // Conditional-store of the updated heap-top.
 8976 // Used during allocation of the shared heap.
 8977 // Sets flag (EQ) on success.
 8978 // implemented using stlxr on AArch64.
 8979 
 8980 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 8981 %{
 8982   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 8983 
 8984   ins_cost(VOLATILE_REF_COST);
 8985 
 8986  // TODO
 8987  // do we need to do a store-conditional release or can we just use a
 8988  // plain store-conditional?
 8989 
 8990   format %{
 8991     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 8992     "cmpw rscratch1, zr\t# EQ on successful write"
 8993   %}
 8994 
 8995   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 8996 
 8997   ins_pipe(pipe_serial);
 8998 %}
 8999 
 9000 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9001 %{
 9002   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9003 
 9004   ins_cost(VOLATILE_REF_COST);
 9005 
 9006   format %{
 9007     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9008     "cmpw rscratch1, zr\t# EQ on successful write"
 9009   %}
 9010 
 9011   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9012 
 9013   ins_pipe(pipe_slow);
 9014 %}
 9015 
 9016 // storeIConditional also has acquire semantics, for no better reason
 9017 // than matching storeLConditional.  At the time of writing this
 9018 // comment storeIConditional was not used anywhere by AArch64.
 9019 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9020 %{
 9021   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9022 
 9023   ins_cost(VOLATILE_REF_COST);
 9024 
 9025   format %{
 9026     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9027     "cmpw rscratch1, zr\t# EQ on successful write"
 9028   %}
 9029 
 9030   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9031 
 9032   ins_pipe(pipe_slow);
 9033 %}
 9034 
 9035 // standard CompareAndSwapX when we are using barriers
 9036 // these have higher priority than the rules selected by a predicate
 9037 
 9038 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9039 // can't match them
 9040 
 9041 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9042 
 9043   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9044   ins_cost(2 * VOLATILE_REF_COST);
 9045 
 9046   effect(KILL cr);
 9047 
 9048   format %{
 9049     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9050     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9051   %}
 9052 
 9053   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9054             aarch64_enc_cset_eq(res));
 9055 
 9056   ins_pipe(pipe_slow);
 9057 %}
 9058 
 9059 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9060 
 9061   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9062   ins_cost(2 * VOLATILE_REF_COST);
 9063 
 9064   effect(KILL cr);
 9065 
 9066   format %{
 9067     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9068     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9069   %}
 9070 
 9071   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9072             aarch64_enc_cset_eq(res));
 9073 
 9074   ins_pipe(pipe_slow);
 9075 %}
 9076 
 9077 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9078 
 9079   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9080   ins_cost(2 * VOLATILE_REF_COST);
 9081 
 9082   effect(KILL cr);
 9083 
 9084  format %{
 9085     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9086     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9087  %}
 9088 
 9089  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9090             aarch64_enc_cset_eq(res));
 9091 
 9092   ins_pipe(pipe_slow);
 9093 %}
 9094 
 9095 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9096 
 9097   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9098   ins_cost(2 * VOLATILE_REF_COST);
 9099 
 9100   effect(KILL cr);
 9101 
 9102  format %{
 9103     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9104     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9105  %}
 9106 
 9107  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9108             aarch64_enc_cset_eq(res));
 9109 
 9110   ins_pipe(pipe_slow);
 9111 %}
 9112 
 9113 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9114 
 9115   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9116   predicate(n->as_LoadStore()->barrier_data() == 0);
 9117   ins_cost(2 * VOLATILE_REF_COST);
 9118 
 9119   effect(KILL cr);
 9120 
 9121  format %{
 9122     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9123     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9124  %}
 9125 
 9126  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9127             aarch64_enc_cset_eq(res));
 9128 
 9129   ins_pipe(pipe_slow);
 9130 %}
 9131 
 9132 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9133 
 9134   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9135   ins_cost(2 * VOLATILE_REF_COST);
 9136 
 9137   effect(KILL cr);
 9138 
 9139  format %{
 9140     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9141     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9142  %}
 9143 
 9144  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9145             aarch64_enc_cset_eq(res));
 9146 
 9147   ins_pipe(pipe_slow);
 9148 %}
 9149 
 9150 // alternative CompareAndSwapX when we are eliding barriers
 9151 
 9152 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9153 
 9154   predicate(needs_acquiring_load_exclusive(n));
 9155   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9156   ins_cost(VOLATILE_REF_COST);
 9157 
 9158   effect(KILL cr);
 9159 
 9160   format %{
 9161     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9162     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9163   %}
 9164 
 9165   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9166             aarch64_enc_cset_eq(res));
 9167 
 9168   ins_pipe(pipe_slow);
 9169 %}
 9170 
 9171 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9172 
 9173   predicate(needs_acquiring_load_exclusive(n));
 9174   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9175   ins_cost(VOLATILE_REF_COST);
 9176 
 9177   effect(KILL cr);
 9178 
 9179   format %{
 9180     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9181     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9182   %}
 9183 
 9184   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9185             aarch64_enc_cset_eq(res));
 9186 
 9187   ins_pipe(pipe_slow);
 9188 %}
 9189 
 9190 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9191 
 9192   predicate(needs_acquiring_load_exclusive(n));
 9193   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9194   ins_cost(VOLATILE_REF_COST);
 9195 
 9196   effect(KILL cr);
 9197 
 9198  format %{
 9199     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9200     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9201  %}
 9202 
 9203  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9204             aarch64_enc_cset_eq(res));
 9205 
 9206   ins_pipe(pipe_slow);
 9207 %}
 9208 
 9209 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9210 
 9211   predicate(needs_acquiring_load_exclusive(n));
 9212   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9213   ins_cost(VOLATILE_REF_COST);
 9214 
 9215   effect(KILL cr);
 9216 
 9217  format %{
 9218     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9219     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9220  %}
 9221 
 9222  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9223             aarch64_enc_cset_eq(res));
 9224 
 9225   ins_pipe(pipe_slow);
 9226 %}
 9227 
 9228 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9229 
 9230   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9231   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9232   ins_cost(VOLATILE_REF_COST);
 9233 
 9234   effect(KILL cr);
 9235 
 9236  format %{
 9237     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9238     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9239  %}
 9240 
 9241  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9242             aarch64_enc_cset_eq(res));
 9243 
 9244   ins_pipe(pipe_slow);
 9245 %}
 9246 
 9247 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9248 
 9249   predicate(needs_acquiring_load_exclusive(n));
 9250   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9251   ins_cost(VOLATILE_REF_COST);
 9252 
 9253   effect(KILL cr);
 9254 
 9255  format %{
 9256     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9257     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9258  %}
 9259 
 9260  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9261             aarch64_enc_cset_eq(res));
 9262 
 9263   ins_pipe(pipe_slow);
 9264 %}
 9265 
 9266 
 9267 // ---------------------------------------------------------------------
 9268 
 9269 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9270 
 9271 // Sundry CAS operations.  Note that release is always true,
 9272 // regardless of the memory ordering of the CAS.  This is because we
 9273 // need the volatile case to be sequentially consistent but there is
 9274 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9275 // can't check the type of memory ordering here, so we always emit a
 9276 // STLXR.
 9277 
 9278 // This section is generated from aarch64_ad_cas.m4
 9279 
 9280 
 9281 
 9282 // This pattern is generated automatically from cas.m4.
 9283 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9284 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9285 
 9286   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9287   ins_cost(2 * VOLATILE_REF_COST);
 9288   effect(TEMP_DEF res, KILL cr);
 9289   format %{
 9290     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9291   %}
 9292   ins_encode %{
 9293     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9294                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9295                /*weak*/ false, $res$$Register);
 9296     __ sxtbw($res$$Register, $res$$Register);
 9297   %}
 9298   ins_pipe(pipe_slow);
 9299 %}
 9300 
 9301 // This pattern is generated automatically from cas.m4.
 9302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9303 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9304 
 9305   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9306   ins_cost(2 * VOLATILE_REF_COST);
 9307   effect(TEMP_DEF res, KILL cr);
 9308   format %{
 9309     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9310   %}
 9311   ins_encode %{
 9312     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9313                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9314                /*weak*/ false, $res$$Register);
 9315     __ sxthw($res$$Register, $res$$Register);
 9316   %}
 9317   ins_pipe(pipe_slow);
 9318 %}
 9319 
 9320 // This pattern is generated automatically from cas.m4.
 9321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9322 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9323 
 9324   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9325   ins_cost(2 * VOLATILE_REF_COST);
 9326   effect(TEMP_DEF res, KILL cr);
 9327   format %{
 9328     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9329   %}
 9330   ins_encode %{
 9331     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9332                Assembler::word, /*acquire*/ false, /*release*/ true,
 9333                /*weak*/ false, $res$$Register);
 9334   %}
 9335   ins_pipe(pipe_slow);
 9336 %}
 9337 
 9338 // This pattern is generated automatically from cas.m4.
 9339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9340 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9341 
 9342   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9343   ins_cost(2 * VOLATILE_REF_COST);
 9344   effect(TEMP_DEF res, KILL cr);
 9345   format %{
 9346     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9347   %}
 9348   ins_encode %{
 9349     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9350                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9351                /*weak*/ false, $res$$Register);
 9352   %}
 9353   ins_pipe(pipe_slow);
 9354 %}
 9355 
 9356 // This pattern is generated automatically from cas.m4.
 9357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9358 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9359 
 9360   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9361   ins_cost(2 * VOLATILE_REF_COST);
 9362   effect(TEMP_DEF res, KILL cr);
 9363   format %{
 9364     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9365   %}
 9366   ins_encode %{
 9367     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9368                Assembler::word, /*acquire*/ false, /*release*/ true,
 9369                /*weak*/ false, $res$$Register);
 9370   %}
 9371   ins_pipe(pipe_slow);
 9372 %}
 9373 
 9374 // This pattern is generated automatically from cas.m4.
 9375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9376 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9377   predicate(n->as_LoadStore()->barrier_data() == 0);
 9378   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9379   ins_cost(2 * VOLATILE_REF_COST);
 9380   effect(TEMP_DEF res, KILL cr);
 9381   format %{
 9382     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9383   %}
 9384   ins_encode %{
 9385     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9386                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9387                /*weak*/ false, $res$$Register);
 9388   %}
 9389   ins_pipe(pipe_slow);
 9390 %}
 9391 
 9392 // This pattern is generated automatically from cas.m4.
 9393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9394 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9395   predicate(needs_acquiring_load_exclusive(n));
 9396   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9397   ins_cost(VOLATILE_REF_COST);
 9398   effect(TEMP_DEF res, KILL cr);
 9399   format %{
 9400     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9401   %}
 9402   ins_encode %{
 9403     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9404                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9405                /*weak*/ false, $res$$Register);
 9406     __ sxtbw($res$$Register, $res$$Register);
 9407   %}
 9408   ins_pipe(pipe_slow);
 9409 %}
 9410 
 9411 // This pattern is generated automatically from cas.m4.
 9412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9413 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9414   predicate(needs_acquiring_load_exclusive(n));
 9415   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9416   ins_cost(VOLATILE_REF_COST);
 9417   effect(TEMP_DEF res, KILL cr);
 9418   format %{
 9419     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9420   %}
 9421   ins_encode %{
 9422     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9423                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9424                /*weak*/ false, $res$$Register);
 9425     __ sxthw($res$$Register, $res$$Register);
 9426   %}
 9427   ins_pipe(pipe_slow);
 9428 %}
 9429 
 9430 // This pattern is generated automatically from cas.m4.
 9431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9432 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9433   predicate(needs_acquiring_load_exclusive(n));
 9434   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9435   ins_cost(VOLATILE_REF_COST);
 9436   effect(TEMP_DEF res, KILL cr);
 9437   format %{
 9438     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9439   %}
 9440   ins_encode %{
 9441     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9442                Assembler::word, /*acquire*/ true, /*release*/ true,
 9443                /*weak*/ false, $res$$Register);
 9444   %}
 9445   ins_pipe(pipe_slow);
 9446 %}
 9447 
 9448 // This pattern is generated automatically from cas.m4.
 9449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9450 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9451   predicate(needs_acquiring_load_exclusive(n));
 9452   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9453   ins_cost(VOLATILE_REF_COST);
 9454   effect(TEMP_DEF res, KILL cr);
 9455   format %{
 9456     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9457   %}
 9458   ins_encode %{
 9459     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9460                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9461                /*weak*/ false, $res$$Register);
 9462   %}
 9463   ins_pipe(pipe_slow);
 9464 %}
 9465 
 9466 // This pattern is generated automatically from cas.m4.
 9467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9468 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9469   predicate(needs_acquiring_load_exclusive(n));
 9470   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9471   ins_cost(VOLATILE_REF_COST);
 9472   effect(TEMP_DEF res, KILL cr);
 9473   format %{
 9474     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9475   %}
 9476   ins_encode %{
 9477     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9478                Assembler::word, /*acquire*/ true, /*release*/ true,
 9479                /*weak*/ false, $res$$Register);
 9480   %}
 9481   ins_pipe(pipe_slow);
 9482 %}
 9483 
 9484 // This pattern is generated automatically from cas.m4.
 9485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9486 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9487   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9488   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9489   ins_cost(VOLATILE_REF_COST);
 9490   effect(TEMP_DEF res, KILL cr);
 9491   format %{
 9492     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9493   %}
 9494   ins_encode %{
 9495     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9496                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9497                /*weak*/ false, $res$$Register);
 9498   %}
 9499   ins_pipe(pipe_slow);
 9500 %}
 9501 
 9502 // This pattern is generated automatically from cas.m4.
 9503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9504 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9505 
 9506   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9507   ins_cost(2 * VOLATILE_REF_COST);
 9508   effect(KILL cr);
 9509   format %{
 9510     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9511     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9512   %}
 9513   ins_encode %{
 9514     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9515                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9516                /*weak*/ true, noreg);
 9517     __ csetw($res$$Register, Assembler::EQ);
 9518   %}
 9519   ins_pipe(pipe_slow);
 9520 %}
 9521 
 9522 // This pattern is generated automatically from cas.m4.
 9523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9524 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9525 
 9526   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9527   ins_cost(2 * VOLATILE_REF_COST);
 9528   effect(KILL cr);
 9529   format %{
 9530     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9531     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9532   %}
 9533   ins_encode %{
 9534     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9535                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9536                /*weak*/ true, noreg);
 9537     __ csetw($res$$Register, Assembler::EQ);
 9538   %}
 9539   ins_pipe(pipe_slow);
 9540 %}
 9541 
 9542 // This pattern is generated automatically from cas.m4.
 9543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9544 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9545 
 9546   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9547   ins_cost(2 * VOLATILE_REF_COST);
 9548   effect(KILL cr);
 9549   format %{
 9550     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9551     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9552   %}
 9553   ins_encode %{
 9554     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9555                Assembler::word, /*acquire*/ false, /*release*/ true,
 9556                /*weak*/ true, noreg);
 9557     __ csetw($res$$Register, Assembler::EQ);
 9558   %}
 9559   ins_pipe(pipe_slow);
 9560 %}
 9561 
 9562 // This pattern is generated automatically from cas.m4.
 9563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9564 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9565 
 9566   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9567   ins_cost(2 * VOLATILE_REF_COST);
 9568   effect(KILL cr);
 9569   format %{
 9570     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9571     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9572   %}
 9573   ins_encode %{
 9574     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9575                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9576                /*weak*/ true, noreg);
 9577     __ csetw($res$$Register, Assembler::EQ);
 9578   %}
 9579   ins_pipe(pipe_slow);
 9580 %}
 9581 
 9582 // This pattern is generated automatically from cas.m4.
 9583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9584 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9585 
 9586   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9587   ins_cost(2 * VOLATILE_REF_COST);
 9588   effect(KILL cr);
 9589   format %{
 9590     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9591     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9592   %}
 9593   ins_encode %{
 9594     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9595                Assembler::word, /*acquire*/ false, /*release*/ true,
 9596                /*weak*/ true, noreg);
 9597     __ csetw($res$$Register, Assembler::EQ);
 9598   %}
 9599   ins_pipe(pipe_slow);
 9600 %}
 9601 
 9602 // This pattern is generated automatically from cas.m4.
 9603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9604 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9605   predicate(n->as_LoadStore()->barrier_data() == 0);
 9606   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9607   ins_cost(2 * VOLATILE_REF_COST);
 9608   effect(KILL cr);
 9609   format %{
 9610     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9611     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9612   %}
 9613   ins_encode %{
 9614     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9615                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9616                /*weak*/ true, noreg);
 9617     __ csetw($res$$Register, Assembler::EQ);
 9618   %}
 9619   ins_pipe(pipe_slow);
 9620 %}
 9621 
 9622 // This pattern is generated automatically from cas.m4.
 9623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9624 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9625   predicate(needs_acquiring_load_exclusive(n));
 9626   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9627   ins_cost(VOLATILE_REF_COST);
 9628   effect(KILL cr);
 9629   format %{
 9630     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9631     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9632   %}
 9633   ins_encode %{
 9634     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9635                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9636                /*weak*/ true, noreg);
 9637     __ csetw($res$$Register, Assembler::EQ);
 9638   %}
 9639   ins_pipe(pipe_slow);
 9640 %}
 9641 
 9642 // This pattern is generated automatically from cas.m4.
 9643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9644 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9645   predicate(needs_acquiring_load_exclusive(n));
 9646   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9647   ins_cost(VOLATILE_REF_COST);
 9648   effect(KILL cr);
 9649   format %{
 9650     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9651     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9652   %}
 9653   ins_encode %{
 9654     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9655                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9656                /*weak*/ true, noreg);
 9657     __ csetw($res$$Register, Assembler::EQ);
 9658   %}
 9659   ins_pipe(pipe_slow);
 9660 %}
 9661 
 9662 // This pattern is generated automatically from cas.m4.
 9663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9664 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9665   predicate(needs_acquiring_load_exclusive(n));
 9666   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9667   ins_cost(VOLATILE_REF_COST);
 9668   effect(KILL cr);
 9669   format %{
 9670     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9671     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9672   %}
 9673   ins_encode %{
 9674     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9675                Assembler::word, /*acquire*/ true, /*release*/ true,
 9676                /*weak*/ true, noreg);
 9677     __ csetw($res$$Register, Assembler::EQ);
 9678   %}
 9679   ins_pipe(pipe_slow);
 9680 %}
 9681 
 9682 // This pattern is generated automatically from cas.m4.
 9683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9684 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9685   predicate(needs_acquiring_load_exclusive(n));
 9686   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9687   ins_cost(VOLATILE_REF_COST);
 9688   effect(KILL cr);
 9689   format %{
 9690     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9691     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9692   %}
 9693   ins_encode %{
 9694     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9695                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9696                /*weak*/ true, noreg);
 9697     __ csetw($res$$Register, Assembler::EQ);
 9698   %}
 9699   ins_pipe(pipe_slow);
 9700 %}
 9701 
 9702 // This pattern is generated automatically from cas.m4.
 9703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9704 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9705   predicate(needs_acquiring_load_exclusive(n));
 9706   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9707   ins_cost(VOLATILE_REF_COST);
 9708   effect(KILL cr);
 9709   format %{
 9710     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9711     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9712   %}
 9713   ins_encode %{
 9714     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9715                Assembler::word, /*acquire*/ true, /*release*/ true,
 9716                /*weak*/ true, noreg);
 9717     __ csetw($res$$Register, Assembler::EQ);
 9718   %}
 9719   ins_pipe(pipe_slow);
 9720 %}
 9721 
 9722 // This pattern is generated automatically from cas.m4.
 9723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9724 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9725   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9726   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9727   ins_cost(VOLATILE_REF_COST);
 9728   effect(KILL cr);
 9729   format %{
 9730     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9731     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9732   %}
 9733   ins_encode %{
 9734     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9735                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9736                /*weak*/ true, noreg);
 9737     __ csetw($res$$Register, Assembler::EQ);
 9738   %}
 9739   ins_pipe(pipe_slow);
 9740 %}
 9741 
 9742 // END This section of the file is automatically generated. Do not edit --------------
 9743 // ---------------------------------------------------------------------
 9744 
 9745 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9746   match(Set prev (GetAndSetI mem newv));
 9747   ins_cost(2 * VOLATILE_REF_COST);
 9748   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9749   ins_encode %{
 9750     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9751   %}
 9752   ins_pipe(pipe_serial);
 9753 %}
 9754 
 9755 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9756   match(Set prev (GetAndSetL mem newv));
 9757   ins_cost(2 * VOLATILE_REF_COST);
 9758   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9759   ins_encode %{
 9760     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9761   %}
 9762   ins_pipe(pipe_serial);
 9763 %}
 9764 
 9765 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9766   match(Set prev (GetAndSetN mem newv));
 9767   ins_cost(2 * VOLATILE_REF_COST);
 9768   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9769   ins_encode %{
 9770     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9771   %}
 9772   ins_pipe(pipe_serial);
 9773 %}
 9774 
 9775 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9776   predicate(n->as_LoadStore()->barrier_data() == 0);
 9777   match(Set prev (GetAndSetP mem newv));
 9778   ins_cost(2 * VOLATILE_REF_COST);
 9779   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9780   ins_encode %{
 9781     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9782   %}
 9783   ins_pipe(pipe_serial);
 9784 %}
 9785 
 9786 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9787   predicate(needs_acquiring_load_exclusive(n));
 9788   match(Set prev (GetAndSetI mem newv));
 9789   ins_cost(VOLATILE_REF_COST);
 9790   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9791   ins_encode %{
 9792     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9793   %}
 9794   ins_pipe(pipe_serial);
 9795 %}
 9796 
 9797 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9798   predicate(needs_acquiring_load_exclusive(n));
 9799   match(Set prev (GetAndSetL mem newv));
 9800   ins_cost(VOLATILE_REF_COST);
 9801   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9802   ins_encode %{
 9803     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9804   %}
 9805   ins_pipe(pipe_serial);
 9806 %}
 9807 
 9808 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9809   predicate(needs_acquiring_load_exclusive(n));
 9810   match(Set prev (GetAndSetN mem newv));
 9811   ins_cost(VOLATILE_REF_COST);
 9812   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9813   ins_encode %{
 9814     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9815   %}
 9816   ins_pipe(pipe_serial);
 9817 %}
 9818 
 9819 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9820   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9821   match(Set prev (GetAndSetP mem newv));
 9822   ins_cost(VOLATILE_REF_COST);
 9823   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9824   ins_encode %{
 9825     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9826   %}
 9827   ins_pipe(pipe_serial);
 9828 %}
 9829 
 9830 
 9831 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9832   match(Set newval (GetAndAddL mem incr));
 9833   ins_cost(2 * VOLATILE_REF_COST + 1);
 9834   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9835   ins_encode %{
 9836     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9837   %}
 9838   ins_pipe(pipe_serial);
 9839 %}
 9840 
 9841 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9842   predicate(n->as_LoadStore()->result_not_used());
 9843   match(Set dummy (GetAndAddL mem incr));
 9844   ins_cost(2 * VOLATILE_REF_COST);
 9845   format %{ "get_and_addL [$mem], $incr" %}
 9846   ins_encode %{
 9847     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9848   %}
 9849   ins_pipe(pipe_serial);
 9850 %}
 9851 
 9852 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9853   match(Set newval (GetAndAddL mem incr));
 9854   ins_cost(2 * VOLATILE_REF_COST + 1);
 9855   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9856   ins_encode %{
 9857     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9858   %}
 9859   ins_pipe(pipe_serial);
 9860 %}
 9861 
 9862 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9863   predicate(n->as_LoadStore()->result_not_used());
 9864   match(Set dummy (GetAndAddL mem incr));
 9865   ins_cost(2 * VOLATILE_REF_COST);
 9866   format %{ "get_and_addL [$mem], $incr" %}
 9867   ins_encode %{
 9868     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9869   %}
 9870   ins_pipe(pipe_serial);
 9871 %}
 9872 
 9873 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9874   match(Set newval (GetAndAddI mem incr));
 9875   ins_cost(2 * VOLATILE_REF_COST + 1);
 9876   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9877   ins_encode %{
 9878     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9879   %}
 9880   ins_pipe(pipe_serial);
 9881 %}
 9882 
 9883 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9884   predicate(n->as_LoadStore()->result_not_used());
 9885   match(Set dummy (GetAndAddI mem incr));
 9886   ins_cost(2 * VOLATILE_REF_COST);
 9887   format %{ "get_and_addI [$mem], $incr" %}
 9888   ins_encode %{
 9889     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9890   %}
 9891   ins_pipe(pipe_serial);
 9892 %}
 9893 
 9894 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9895   match(Set newval (GetAndAddI mem incr));
 9896   ins_cost(2 * VOLATILE_REF_COST + 1);
 9897   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9898   ins_encode %{
 9899     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9900   %}
 9901   ins_pipe(pipe_serial);
 9902 %}
 9903 
 9904 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9905   predicate(n->as_LoadStore()->result_not_used());
 9906   match(Set dummy (GetAndAddI mem incr));
 9907   ins_cost(2 * VOLATILE_REF_COST);
 9908   format %{ "get_and_addI [$mem], $incr" %}
 9909   ins_encode %{
 9910     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9911   %}
 9912   ins_pipe(pipe_serial);
 9913 %}
 9914 
 9915 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9916   predicate(needs_acquiring_load_exclusive(n));
 9917   match(Set newval (GetAndAddL mem incr));
 9918   ins_cost(VOLATILE_REF_COST + 1);
 9919   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9920   ins_encode %{
 9921     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9922   %}
 9923   ins_pipe(pipe_serial);
 9924 %}
 9925 
 9926 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9927   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9928   match(Set dummy (GetAndAddL mem incr));
 9929   ins_cost(VOLATILE_REF_COST);
 9930   format %{ "get_and_addL_acq [$mem], $incr" %}
 9931   ins_encode %{
 9932     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9933   %}
 9934   ins_pipe(pipe_serial);
 9935 %}
 9936 
 9937 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9938   predicate(needs_acquiring_load_exclusive(n));
 9939   match(Set newval (GetAndAddL mem incr));
 9940   ins_cost(VOLATILE_REF_COST + 1);
 9941   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9942   ins_encode %{
 9943     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9944   %}
 9945   ins_pipe(pipe_serial);
 9946 %}
 9947 
 9948 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9949   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9950   match(Set dummy (GetAndAddL mem incr));
 9951   ins_cost(VOLATILE_REF_COST);
 9952   format %{ "get_and_addL_acq [$mem], $incr" %}
 9953   ins_encode %{
 9954     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9955   %}
 9956   ins_pipe(pipe_serial);
 9957 %}
 9958 
 9959 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9960   predicate(needs_acquiring_load_exclusive(n));
 9961   match(Set newval (GetAndAddI mem incr));
 9962   ins_cost(VOLATILE_REF_COST + 1);
 9963   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9964   ins_encode %{
 9965     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9966   %}
 9967   ins_pipe(pipe_serial);
 9968 %}
 9969 
 9970 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9971   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9972   match(Set dummy (GetAndAddI mem incr));
 9973   ins_cost(VOLATILE_REF_COST);
 9974   format %{ "get_and_addI_acq [$mem], $incr" %}
 9975   ins_encode %{
 9976     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9977   %}
 9978   ins_pipe(pipe_serial);
 9979 %}
 9980 
 9981 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9982   predicate(needs_acquiring_load_exclusive(n));
 9983   match(Set newval (GetAndAddI mem incr));
 9984   ins_cost(VOLATILE_REF_COST + 1);
 9985   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9986   ins_encode %{
 9987     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9988   %}
 9989   ins_pipe(pipe_serial);
 9990 %}
 9991 
 9992 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9993   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9994   match(Set dummy (GetAndAddI mem incr));
 9995   ins_cost(VOLATILE_REF_COST);
 9996   format %{ "get_and_addI_acq [$mem], $incr" %}
 9997   ins_encode %{
 9998     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9999   %}
10000   ins_pipe(pipe_serial);
10001 %}
10002 
10003 // Manifest a CmpL result in an integer register.
10004 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10005 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10006 %{
10007   match(Set dst (CmpL3 src1 src2));
10008   effect(KILL flags);
10009 
10010   ins_cost(INSN_COST * 6);
10011   format %{
10012       "cmp $src1, $src2"
10013       "csetw $dst, ne"
10014       "cnegw $dst, lt"
10015   %}
10016   // format %{ "CmpL3 $dst, $src1, $src2" %}
10017   ins_encode %{
10018     __ cmp($src1$$Register, $src2$$Register);
10019     __ csetw($dst$$Register, Assembler::NE);
10020     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10021   %}
10022 
10023   ins_pipe(pipe_class_default);
10024 %}
10025 
10026 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10027 %{
10028   match(Set dst (CmpL3 src1 src2));
10029   effect(KILL flags);
10030 
10031   ins_cost(INSN_COST * 6);
10032   format %{
10033       "cmp $src1, $src2"
10034       "csetw $dst, ne"
10035       "cnegw $dst, lt"
10036   %}
10037   ins_encode %{
10038     int32_t con = (int32_t)$src2$$constant;
10039      if (con < 0) {
10040       __ adds(zr, $src1$$Register, -con);
10041     } else {
10042       __ subs(zr, $src1$$Register, con);
10043     }
10044     __ csetw($dst$$Register, Assembler::NE);
10045     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10046   %}
10047 
10048   ins_pipe(pipe_class_default);
10049 %}
10050 
10051 // ============================================================================
10052 // Conditional Move Instructions
10053 
10054 // n.b. we have identical rules for both a signed compare op (cmpOp)
10055 // and an unsigned compare op (cmpOpU). it would be nice if we could
10056 // define an op class which merged both inputs and use it to type the
10057 // argument to a single rule. unfortunatelyt his fails because the
10058 // opclass does not live up to the COND_INTER interface of its
10059 // component operands. When the generic code tries to negate the
10060 // operand it ends up running the generci Machoper::negate method
10061 // which throws a ShouldNotHappen. So, we have to provide two flavours
10062 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10063 
10064 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10065   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10066 
10067   ins_cost(INSN_COST * 2);
10068   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10069 
10070   ins_encode %{
10071     __ cselw(as_Register($dst$$reg),
10072              as_Register($src2$$reg),
10073              as_Register($src1$$reg),
10074              (Assembler::Condition)$cmp$$cmpcode);
10075   %}
10076 
10077   ins_pipe(icond_reg_reg);
10078 %}
10079 
10080 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10081   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10082 
10083   ins_cost(INSN_COST * 2);
10084   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10085 
10086   ins_encode %{
10087     __ cselw(as_Register($dst$$reg),
10088              as_Register($src2$$reg),
10089              as_Register($src1$$reg),
10090              (Assembler::Condition)$cmp$$cmpcode);
10091   %}
10092 
10093   ins_pipe(icond_reg_reg);
10094 %}
10095 
10096 // special cases where one arg is zero
10097 
10098 // n.b. this is selected in preference to the rule above because it
10099 // avoids loading constant 0 into a source register
10100 
10101 // TODO
10102 // we ought only to be able to cull one of these variants as the ideal
10103 // transforms ought always to order the zero consistently (to left/right?)
10104 
10105 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10106   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10107 
10108   ins_cost(INSN_COST * 2);
10109   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10110 
10111   ins_encode %{
10112     __ cselw(as_Register($dst$$reg),
10113              as_Register($src$$reg),
10114              zr,
10115              (Assembler::Condition)$cmp$$cmpcode);
10116   %}
10117 
10118   ins_pipe(icond_reg);
10119 %}
10120 
10121 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10122   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10123 
10124   ins_cost(INSN_COST * 2);
10125   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10126 
10127   ins_encode %{
10128     __ cselw(as_Register($dst$$reg),
10129              as_Register($src$$reg),
10130              zr,
10131              (Assembler::Condition)$cmp$$cmpcode);
10132   %}
10133 
10134   ins_pipe(icond_reg);
10135 %}
10136 
10137 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10138   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10139 
10140   ins_cost(INSN_COST * 2);
10141   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10142 
10143   ins_encode %{
10144     __ cselw(as_Register($dst$$reg),
10145              zr,
10146              as_Register($src$$reg),
10147              (Assembler::Condition)$cmp$$cmpcode);
10148   %}
10149 
10150   ins_pipe(icond_reg);
10151 %}
10152 
10153 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10154   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10155 
10156   ins_cost(INSN_COST * 2);
10157   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10158 
10159   ins_encode %{
10160     __ cselw(as_Register($dst$$reg),
10161              zr,
10162              as_Register($src$$reg),
10163              (Assembler::Condition)$cmp$$cmpcode);
10164   %}
10165 
10166   ins_pipe(icond_reg);
10167 %}
10168 
10169 // special case for creating a boolean 0 or 1
10170 
10171 // n.b. this is selected in preference to the rule above because it
10172 // avoids loading constants 0 and 1 into a source register
10173 
10174 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10175   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10176 
10177   ins_cost(INSN_COST * 2);
10178   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10179 
10180   ins_encode %{
10181     // equivalently
10182     // cset(as_Register($dst$$reg),
10183     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10184     __ csincw(as_Register($dst$$reg),
10185              zr,
10186              zr,
10187              (Assembler::Condition)$cmp$$cmpcode);
10188   %}
10189 
10190   ins_pipe(icond_none);
10191 %}
10192 
10193 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10194   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10195 
10196   ins_cost(INSN_COST * 2);
10197   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10198 
10199   ins_encode %{
10200     // equivalently
10201     // cset(as_Register($dst$$reg),
10202     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10203     __ csincw(as_Register($dst$$reg),
10204              zr,
10205              zr,
10206              (Assembler::Condition)$cmp$$cmpcode);
10207   %}
10208 
10209   ins_pipe(icond_none);
10210 %}
10211 
10212 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10213   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10214 
10215   ins_cost(INSN_COST * 2);
10216   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10217 
10218   ins_encode %{
10219     __ csel(as_Register($dst$$reg),
10220             as_Register($src2$$reg),
10221             as_Register($src1$$reg),
10222             (Assembler::Condition)$cmp$$cmpcode);
10223   %}
10224 
10225   ins_pipe(icond_reg_reg);
10226 %}
10227 
10228 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10229   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10230 
10231   ins_cost(INSN_COST * 2);
10232   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10233 
10234   ins_encode %{
10235     __ csel(as_Register($dst$$reg),
10236             as_Register($src2$$reg),
10237             as_Register($src1$$reg),
10238             (Assembler::Condition)$cmp$$cmpcode);
10239   %}
10240 
10241   ins_pipe(icond_reg_reg);
10242 %}
10243 
10244 // special cases where one arg is zero
10245 
10246 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10247   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10248 
10249   ins_cost(INSN_COST * 2);
10250   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10251 
10252   ins_encode %{
10253     __ csel(as_Register($dst$$reg),
10254             zr,
10255             as_Register($src$$reg),
10256             (Assembler::Condition)$cmp$$cmpcode);
10257   %}
10258 
10259   ins_pipe(icond_reg);
10260 %}
10261 
10262 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10263   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10264 
10265   ins_cost(INSN_COST * 2);
10266   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10267 
10268   ins_encode %{
10269     __ csel(as_Register($dst$$reg),
10270             zr,
10271             as_Register($src$$reg),
10272             (Assembler::Condition)$cmp$$cmpcode);
10273   %}
10274 
10275   ins_pipe(icond_reg);
10276 %}
10277 
10278 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10279   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10280 
10281   ins_cost(INSN_COST * 2);
10282   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10283 
10284   ins_encode %{
10285     __ csel(as_Register($dst$$reg),
10286             as_Register($src$$reg),
10287             zr,
10288             (Assembler::Condition)$cmp$$cmpcode);
10289   %}
10290 
10291   ins_pipe(icond_reg);
10292 %}
10293 
10294 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10295   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10296 
10297   ins_cost(INSN_COST * 2);
10298   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10299 
10300   ins_encode %{
10301     __ csel(as_Register($dst$$reg),
10302             as_Register($src$$reg),
10303             zr,
10304             (Assembler::Condition)$cmp$$cmpcode);
10305   %}
10306 
10307   ins_pipe(icond_reg);
10308 %}
10309 
10310 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10311   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10312 
10313   ins_cost(INSN_COST * 2);
10314   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10315 
10316   ins_encode %{
10317     __ csel(as_Register($dst$$reg),
10318             as_Register($src2$$reg),
10319             as_Register($src1$$reg),
10320             (Assembler::Condition)$cmp$$cmpcode);
10321   %}
10322 
10323   ins_pipe(icond_reg_reg);
10324 %}
10325 
10326 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10327   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10328 
10329   ins_cost(INSN_COST * 2);
10330   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10331 
10332   ins_encode %{
10333     __ csel(as_Register($dst$$reg),
10334             as_Register($src2$$reg),
10335             as_Register($src1$$reg),
10336             (Assembler::Condition)$cmp$$cmpcode);
10337   %}
10338 
10339   ins_pipe(icond_reg_reg);
10340 %}
10341 
10342 // special cases where one arg is zero
10343 
10344 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10345   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10346 
10347   ins_cost(INSN_COST * 2);
10348   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10349 
10350   ins_encode %{
10351     __ csel(as_Register($dst$$reg),
10352             zr,
10353             as_Register($src$$reg),
10354             (Assembler::Condition)$cmp$$cmpcode);
10355   %}
10356 
10357   ins_pipe(icond_reg);
10358 %}
10359 
10360 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10361   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10362 
10363   ins_cost(INSN_COST * 2);
10364   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10365 
10366   ins_encode %{
10367     __ csel(as_Register($dst$$reg),
10368             zr,
10369             as_Register($src$$reg),
10370             (Assembler::Condition)$cmp$$cmpcode);
10371   %}
10372 
10373   ins_pipe(icond_reg);
10374 %}
10375 
10376 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10377   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10378 
10379   ins_cost(INSN_COST * 2);
10380   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10381 
10382   ins_encode %{
10383     __ csel(as_Register($dst$$reg),
10384             as_Register($src$$reg),
10385             zr,
10386             (Assembler::Condition)$cmp$$cmpcode);
10387   %}
10388 
10389   ins_pipe(icond_reg);
10390 %}
10391 
10392 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10393   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10394 
10395   ins_cost(INSN_COST * 2);
10396   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10397 
10398   ins_encode %{
10399     __ csel(as_Register($dst$$reg),
10400             as_Register($src$$reg),
10401             zr,
10402             (Assembler::Condition)$cmp$$cmpcode);
10403   %}
10404 
10405   ins_pipe(icond_reg);
10406 %}
10407 
10408 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10409   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10410 
10411   ins_cost(INSN_COST * 2);
10412   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10413 
10414   ins_encode %{
10415     __ cselw(as_Register($dst$$reg),
10416              as_Register($src2$$reg),
10417              as_Register($src1$$reg),
10418              (Assembler::Condition)$cmp$$cmpcode);
10419   %}
10420 
10421   ins_pipe(icond_reg_reg);
10422 %}
10423 
10424 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10425   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10426 
10427   ins_cost(INSN_COST * 2);
10428   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10429 
10430   ins_encode %{
10431     __ cselw(as_Register($dst$$reg),
10432              as_Register($src2$$reg),
10433              as_Register($src1$$reg),
10434              (Assembler::Condition)$cmp$$cmpcode);
10435   %}
10436 
10437   ins_pipe(icond_reg_reg);
10438 %}
10439 
10440 // special cases where one arg is zero
10441 
10442 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10443   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10444 
10445   ins_cost(INSN_COST * 2);
10446   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10447 
10448   ins_encode %{
10449     __ cselw(as_Register($dst$$reg),
10450              zr,
10451              as_Register($src$$reg),
10452              (Assembler::Condition)$cmp$$cmpcode);
10453   %}
10454 
10455   ins_pipe(icond_reg);
10456 %}
10457 
10458 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10459   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10460 
10461   ins_cost(INSN_COST * 2);
10462   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10463 
10464   ins_encode %{
10465     __ cselw(as_Register($dst$$reg),
10466              zr,
10467              as_Register($src$$reg),
10468              (Assembler::Condition)$cmp$$cmpcode);
10469   %}
10470 
10471   ins_pipe(icond_reg);
10472 %}
10473 
10474 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10475   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10476 
10477   ins_cost(INSN_COST * 2);
10478   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10479 
10480   ins_encode %{
10481     __ cselw(as_Register($dst$$reg),
10482              as_Register($src$$reg),
10483              zr,
10484              (Assembler::Condition)$cmp$$cmpcode);
10485   %}
10486 
10487   ins_pipe(icond_reg);
10488 %}
10489 
10490 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10491   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10492 
10493   ins_cost(INSN_COST * 2);
10494   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10495 
10496   ins_encode %{
10497     __ cselw(as_Register($dst$$reg),
10498              as_Register($src$$reg),
10499              zr,
10500              (Assembler::Condition)$cmp$$cmpcode);
10501   %}
10502 
10503   ins_pipe(icond_reg);
10504 %}
10505 
10506 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10507 %{
10508   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10509 
10510   ins_cost(INSN_COST * 3);
10511 
10512   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10513   ins_encode %{
10514     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10515     __ fcsels(as_FloatRegister($dst$$reg),
10516               as_FloatRegister($src2$$reg),
10517               as_FloatRegister($src1$$reg),
10518               cond);
10519   %}
10520 
10521   ins_pipe(fp_cond_reg_reg_s);
10522 %}
10523 
10524 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10525 %{
10526   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10527 
10528   ins_cost(INSN_COST * 3);
10529 
10530   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10531   ins_encode %{
10532     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10533     __ fcsels(as_FloatRegister($dst$$reg),
10534               as_FloatRegister($src2$$reg),
10535               as_FloatRegister($src1$$reg),
10536               cond);
10537   %}
10538 
10539   ins_pipe(fp_cond_reg_reg_s);
10540 %}
10541 
10542 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10543 %{
10544   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10545 
10546   ins_cost(INSN_COST * 3);
10547 
10548   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10549   ins_encode %{
10550     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10551     __ fcseld(as_FloatRegister($dst$$reg),
10552               as_FloatRegister($src2$$reg),
10553               as_FloatRegister($src1$$reg),
10554               cond);
10555   %}
10556 
10557   ins_pipe(fp_cond_reg_reg_d);
10558 %}
10559 
10560 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10561 %{
10562   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10563 
10564   ins_cost(INSN_COST * 3);
10565 
10566   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10567   ins_encode %{
10568     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10569     __ fcseld(as_FloatRegister($dst$$reg),
10570               as_FloatRegister($src2$$reg),
10571               as_FloatRegister($src1$$reg),
10572               cond);
10573   %}
10574 
10575   ins_pipe(fp_cond_reg_reg_d);
10576 %}
10577 
10578 // ============================================================================
10579 // Arithmetic Instructions
10580 //
10581 
10582 // Integer Addition
10583 
10584 // TODO
10585 // these currently employ operations which do not set CR and hence are
10586 // not flagged as killing CR but we would like to isolate the cases
10587 // where we want to set flags from those where we don't. need to work
10588 // out how to do that.
10589 
10590 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10591   match(Set dst (AddI src1 src2));
10592 
10593   ins_cost(INSN_COST);
10594   format %{ "addw  $dst, $src1, $src2" %}
10595 
10596   ins_encode %{
10597     __ addw(as_Register($dst$$reg),
10598             as_Register($src1$$reg),
10599             as_Register($src2$$reg));
10600   %}
10601 
10602   ins_pipe(ialu_reg_reg);
10603 %}
10604 
10605 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10606   match(Set dst (AddI src1 src2));
10607 
10608   ins_cost(INSN_COST);
10609   format %{ "addw $dst, $src1, $src2" %}
10610 
10611   // use opcode to indicate that this is an add not a sub
10612   opcode(0x0);
10613 
10614   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10615 
10616   ins_pipe(ialu_reg_imm);
10617 %}
10618 
10619 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10620   match(Set dst (AddI (ConvL2I src1) src2));
10621 
10622   ins_cost(INSN_COST);
10623   format %{ "addw $dst, $src1, $src2" %}
10624 
10625   // use opcode to indicate that this is an add not a sub
10626   opcode(0x0);
10627 
10628   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10629 
10630   ins_pipe(ialu_reg_imm);
10631 %}
10632 
10633 // Pointer Addition
10634 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10635   match(Set dst (AddP src1 src2));
10636 
10637   ins_cost(INSN_COST);
10638   format %{ "add $dst, $src1, $src2\t# ptr" %}
10639 
10640   ins_encode %{
10641     __ add(as_Register($dst$$reg),
10642            as_Register($src1$$reg),
10643            as_Register($src2$$reg));
10644   %}
10645 
10646   ins_pipe(ialu_reg_reg);
10647 %}
10648 
10649 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10650   match(Set dst (AddP src1 (ConvI2L src2)));
10651 
10652   ins_cost(1.9 * INSN_COST);
10653   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10654 
10655   ins_encode %{
10656     __ add(as_Register($dst$$reg),
10657            as_Register($src1$$reg),
10658            as_Register($src2$$reg), ext::sxtw);
10659   %}
10660 
10661   ins_pipe(ialu_reg_reg);
10662 %}
10663 
10664 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10665   match(Set dst (AddP src1 (LShiftL src2 scale)));
10666 
10667   ins_cost(1.9 * INSN_COST);
10668   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10669 
10670   ins_encode %{
10671     __ lea(as_Register($dst$$reg),
10672            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10673                    Address::lsl($scale$$constant)));
10674   %}
10675 
10676   ins_pipe(ialu_reg_reg_shift);
10677 %}
10678 
10679 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10680   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10681 
10682   ins_cost(1.9 * INSN_COST);
10683   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10684 
10685   ins_encode %{
10686     __ lea(as_Register($dst$$reg),
10687            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10688                    Address::sxtw($scale$$constant)));
10689   %}
10690 
10691   ins_pipe(ialu_reg_reg_shift);
10692 %}
10693 
10694 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10695   match(Set dst (LShiftL (ConvI2L src) scale));
10696 
10697   ins_cost(INSN_COST);
10698   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10699 
10700   ins_encode %{
10701     __ sbfiz(as_Register($dst$$reg),
10702           as_Register($src$$reg),
10703           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10704   %}
10705 
10706   ins_pipe(ialu_reg_shift);
10707 %}
10708 
10709 // Pointer Immediate Addition
10710 // n.b. this needs to be more expensive than using an indirect memory
10711 // operand
10712 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10713   match(Set dst (AddP src1 src2));
10714 
10715   ins_cost(INSN_COST);
10716   format %{ "add $dst, $src1, $src2\t# ptr" %}
10717 
10718   // use opcode to indicate that this is an add not a sub
10719   opcode(0x0);
10720 
10721   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10722 
10723   ins_pipe(ialu_reg_imm);
10724 %}
10725 
10726 // Long Addition
10727 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10728 
10729   match(Set dst (AddL src1 src2));
10730 
10731   ins_cost(INSN_COST);
10732   format %{ "add  $dst, $src1, $src2" %}
10733 
10734   ins_encode %{
10735     __ add(as_Register($dst$$reg),
10736            as_Register($src1$$reg),
10737            as_Register($src2$$reg));
10738   %}
10739 
10740   ins_pipe(ialu_reg_reg);
10741 %}
10742 
10743 // No constant pool entries requiredLong Immediate Addition.
10744 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10745   match(Set dst (AddL src1 src2));
10746 
10747   ins_cost(INSN_COST);
10748   format %{ "add $dst, $src1, $src2" %}
10749 
10750   // use opcode to indicate that this is an add not a sub
10751   opcode(0x0);
10752 
10753   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10754 
10755   ins_pipe(ialu_reg_imm);
10756 %}
10757 
10758 // Integer Subtraction
10759 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10760   match(Set dst (SubI src1 src2));
10761 
10762   ins_cost(INSN_COST);
10763   format %{ "subw  $dst, $src1, $src2" %}
10764 
10765   ins_encode %{
10766     __ subw(as_Register($dst$$reg),
10767             as_Register($src1$$reg),
10768             as_Register($src2$$reg));
10769   %}
10770 
10771   ins_pipe(ialu_reg_reg);
10772 %}
10773 
10774 // Immediate Subtraction
10775 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10776   match(Set dst (SubI src1 src2));
10777 
10778   ins_cost(INSN_COST);
10779   format %{ "subw $dst, $src1, $src2" %}
10780 
10781   // use opcode to indicate that this is a sub not an add
10782   opcode(0x1);
10783 
10784   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10785 
10786   ins_pipe(ialu_reg_imm);
10787 %}
10788 
10789 // Long Subtraction
10790 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10791 
10792   match(Set dst (SubL src1 src2));
10793 
10794   ins_cost(INSN_COST);
10795   format %{ "sub  $dst, $src1, $src2" %}
10796 
10797   ins_encode %{
10798     __ sub(as_Register($dst$$reg),
10799            as_Register($src1$$reg),
10800            as_Register($src2$$reg));
10801   %}
10802 
10803   ins_pipe(ialu_reg_reg);
10804 %}
10805 
10806 // No constant pool entries requiredLong Immediate Subtraction.
10807 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10808   match(Set dst (SubL src1 src2));
10809 
10810   ins_cost(INSN_COST);
10811   format %{ "sub$dst, $src1, $src2" %}
10812 
10813   // use opcode to indicate that this is a sub not an add
10814   opcode(0x1);
10815 
10816   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10817 
10818   ins_pipe(ialu_reg_imm);
10819 %}
10820 
10821 // Integer Negation (special case for sub)
10822 
10823 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10824   match(Set dst (SubI zero src));
10825 
10826   ins_cost(INSN_COST);
10827   format %{ "negw $dst, $src\t# int" %}
10828 
10829   ins_encode %{
10830     __ negw(as_Register($dst$$reg),
10831             as_Register($src$$reg));
10832   %}
10833 
10834   ins_pipe(ialu_reg);
10835 %}
10836 
10837 // Long Negation
10838 
10839 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10840   match(Set dst (SubL zero src));
10841 
10842   ins_cost(INSN_COST);
10843   format %{ "neg $dst, $src\t# long" %}
10844 
10845   ins_encode %{
10846     __ neg(as_Register($dst$$reg),
10847            as_Register($src$$reg));
10848   %}
10849 
10850   ins_pipe(ialu_reg);
10851 %}
10852 
10853 // Integer Multiply
10854 
10855 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10856   match(Set dst (MulI src1 src2));
10857 
10858   ins_cost(INSN_COST * 3);
10859   format %{ "mulw  $dst, $src1, $src2" %}
10860 
10861   ins_encode %{
10862     __ mulw(as_Register($dst$$reg),
10863             as_Register($src1$$reg),
10864             as_Register($src2$$reg));
10865   %}
10866 
10867   ins_pipe(imul_reg_reg);
10868 %}
10869 
10870 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10871   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10872 
10873   ins_cost(INSN_COST * 3);
10874   format %{ "smull  $dst, $src1, $src2" %}
10875 
10876   ins_encode %{
10877     __ smull(as_Register($dst$$reg),
10878              as_Register($src1$$reg),
10879              as_Register($src2$$reg));
10880   %}
10881 
10882   ins_pipe(imul_reg_reg);
10883 %}
10884 
10885 // Long Multiply
10886 
10887 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10888   match(Set dst (MulL src1 src2));
10889 
10890   ins_cost(INSN_COST * 5);
10891   format %{ "mul  $dst, $src1, $src2" %}
10892 
10893   ins_encode %{
10894     __ mul(as_Register($dst$$reg),
10895            as_Register($src1$$reg),
10896            as_Register($src2$$reg));
10897   %}
10898 
10899   ins_pipe(lmul_reg_reg);
10900 %}
10901 
10902 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10903 %{
10904   match(Set dst (MulHiL src1 src2));
10905 
10906   ins_cost(INSN_COST * 7);
10907   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10908 
10909   ins_encode %{
10910     __ smulh(as_Register($dst$$reg),
10911              as_Register($src1$$reg),
10912              as_Register($src2$$reg));
10913   %}
10914 
10915   ins_pipe(lmul_reg_reg);
10916 %}
10917 
10918 // Combined Integer Multiply & Add/Sub
10919 
10920 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10921   match(Set dst (AddI src3 (MulI src1 src2)));
10922 
10923   ins_cost(INSN_COST * 3);
10924   format %{ "madd  $dst, $src1, $src2, $src3" %}
10925 
10926   ins_encode %{
10927     __ maddw(as_Register($dst$$reg),
10928              as_Register($src1$$reg),
10929              as_Register($src2$$reg),
10930              as_Register($src3$$reg));
10931   %}
10932 
10933   ins_pipe(imac_reg_reg);
10934 %}
10935 
10936 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10937   match(Set dst (SubI src3 (MulI src1 src2)));
10938 
10939   ins_cost(INSN_COST * 3);
10940   format %{ "msub  $dst, $src1, $src2, $src3" %}
10941 
10942   ins_encode %{
10943     __ msubw(as_Register($dst$$reg),
10944              as_Register($src1$$reg),
10945              as_Register($src2$$reg),
10946              as_Register($src3$$reg));
10947   %}
10948 
10949   ins_pipe(imac_reg_reg);
10950 %}
10951 
10952 // Combined Integer Multiply & Neg
10953 
10954 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10955   match(Set dst (MulI (SubI zero src1) src2));
10956   match(Set dst (MulI src1 (SubI zero src2)));
10957 
10958   ins_cost(INSN_COST * 3);
10959   format %{ "mneg  $dst, $src1, $src2" %}
10960 
10961   ins_encode %{
10962     __ mnegw(as_Register($dst$$reg),
10963              as_Register($src1$$reg),
10964              as_Register($src2$$reg));
10965   %}
10966 
10967   ins_pipe(imac_reg_reg);
10968 %}
10969 
10970 // Combined Long Multiply & Add/Sub
10971 
10972 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10973   match(Set dst (AddL src3 (MulL src1 src2)));
10974 
10975   ins_cost(INSN_COST * 5);
10976   format %{ "madd  $dst, $src1, $src2, $src3" %}
10977 
10978   ins_encode %{
10979     __ madd(as_Register($dst$$reg),
10980             as_Register($src1$$reg),
10981             as_Register($src2$$reg),
10982             as_Register($src3$$reg));
10983   %}
10984 
10985   ins_pipe(lmac_reg_reg);
10986 %}
10987 
10988 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10989   match(Set dst (SubL src3 (MulL src1 src2)));
10990 
10991   ins_cost(INSN_COST * 5);
10992   format %{ "msub  $dst, $src1, $src2, $src3" %}
10993 
10994   ins_encode %{
10995     __ msub(as_Register($dst$$reg),
10996             as_Register($src1$$reg),
10997             as_Register($src2$$reg),
10998             as_Register($src3$$reg));
10999   %}
11000 
11001   ins_pipe(lmac_reg_reg);
11002 %}
11003 
11004 // Combined Long Multiply & Neg
11005 
11006 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11007   match(Set dst (MulL (SubL zero src1) src2));
11008   match(Set dst (MulL src1 (SubL zero src2)));
11009 
11010   ins_cost(INSN_COST * 5);
11011   format %{ "mneg  $dst, $src1, $src2" %}
11012 
11013   ins_encode %{
11014     __ mneg(as_Register($dst$$reg),
11015             as_Register($src1$$reg),
11016             as_Register($src2$$reg));
11017   %}
11018 
11019   ins_pipe(lmac_reg_reg);
11020 %}
11021 
11022 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11023 
11024 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11025   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11026 
11027   ins_cost(INSN_COST * 3);
11028   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11029 
11030   ins_encode %{
11031     __ smaddl(as_Register($dst$$reg),
11032               as_Register($src1$$reg),
11033               as_Register($src2$$reg),
11034               as_Register($src3$$reg));
11035   %}
11036 
11037   ins_pipe(imac_reg_reg);
11038 %}
11039 
11040 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11041   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11042 
11043   ins_cost(INSN_COST * 3);
11044   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11045 
11046   ins_encode %{
11047     __ smsubl(as_Register($dst$$reg),
11048               as_Register($src1$$reg),
11049               as_Register($src2$$reg),
11050               as_Register($src3$$reg));
11051   %}
11052 
11053   ins_pipe(imac_reg_reg);
11054 %}
11055 
11056 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11057   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11058   match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
11059 
11060   ins_cost(INSN_COST * 3);
11061   format %{ "smnegl  $dst, $src1, $src2" %}
11062 
11063   ins_encode %{
11064     __ smnegl(as_Register($dst$$reg),
11065               as_Register($src1$$reg),
11066               as_Register($src2$$reg));
11067   %}
11068 
11069   ins_pipe(imac_reg_reg);
11070 %}
11071 
11072 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11073 
11074 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11075   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11076 
11077   ins_cost(INSN_COST * 5);
11078   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11079             "maddw $dst, $src3, $src4, rscratch1" %}
11080 
11081   ins_encode %{
11082     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11083     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11084 
11085   ins_pipe(imac_reg_reg);
11086 %}
11087 
11088 // Integer Divide
11089 
11090 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11091   match(Set dst (DivI src1 src2));
11092 
11093   ins_cost(INSN_COST * 19);
11094   format %{ "sdivw  $dst, $src1, $src2" %}
11095 
11096   ins_encode(aarch64_enc_divw(dst, src1, src2));
11097   ins_pipe(idiv_reg_reg);
11098 %}
11099 
11100 // Long Divide
11101 
11102 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11103   match(Set dst (DivL src1 src2));
11104 
11105   ins_cost(INSN_COST * 35);
11106   format %{ "sdiv   $dst, $src1, $src2" %}
11107 
11108   ins_encode(aarch64_enc_div(dst, src1, src2));
11109   ins_pipe(ldiv_reg_reg);
11110 %}
11111 
11112 // Integer Remainder
11113 
11114 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11115   match(Set dst (ModI src1 src2));
11116 
11117   ins_cost(INSN_COST * 22);
11118   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11119             "msubw($dst, rscratch1, $src2, $src1" %}
11120 
11121   ins_encode(aarch64_enc_modw(dst, src1, src2));
11122   ins_pipe(idiv_reg_reg);
11123 %}
11124 
11125 // Long Remainder
11126 
11127 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11128   match(Set dst (ModL src1 src2));
11129 
11130   ins_cost(INSN_COST * 38);
11131   format %{ "sdiv   rscratch1, $src1, $src2\n"
11132             "msub($dst, rscratch1, $src2, $src1" %}
11133 
11134   ins_encode(aarch64_enc_mod(dst, src1, src2));
11135   ins_pipe(ldiv_reg_reg);
11136 %}
11137 
11138 // Integer Shifts
11139 
11140 // Shift Left Register
11141 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11142   match(Set dst (LShiftI src1 src2));
11143 
11144   ins_cost(INSN_COST * 2);
11145   format %{ "lslvw  $dst, $src1, $src2" %}
11146 
11147   ins_encode %{
11148     __ lslvw(as_Register($dst$$reg),
11149              as_Register($src1$$reg),
11150              as_Register($src2$$reg));
11151   %}
11152 
11153   ins_pipe(ialu_reg_reg_vshift);
11154 %}
11155 
11156 // Shift Left Immediate
11157 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11158   match(Set dst (LShiftI src1 src2));
11159 
11160   ins_cost(INSN_COST);
11161   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11162 
11163   ins_encode %{
11164     __ lslw(as_Register($dst$$reg),
11165             as_Register($src1$$reg),
11166             $src2$$constant & 0x1f);
11167   %}
11168 
11169   ins_pipe(ialu_reg_shift);
11170 %}
11171 
11172 // Shift Right Logical Register
11173 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11174   match(Set dst (URShiftI src1 src2));
11175 
11176   ins_cost(INSN_COST * 2);
11177   format %{ "lsrvw  $dst, $src1, $src2" %}
11178 
11179   ins_encode %{
11180     __ lsrvw(as_Register($dst$$reg),
11181              as_Register($src1$$reg),
11182              as_Register($src2$$reg));
11183   %}
11184 
11185   ins_pipe(ialu_reg_reg_vshift);
11186 %}
11187 
11188 // Shift Right Logical Immediate
11189 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11190   match(Set dst (URShiftI src1 src2));
11191 
11192   ins_cost(INSN_COST);
11193   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11194 
11195   ins_encode %{
11196     __ lsrw(as_Register($dst$$reg),
11197             as_Register($src1$$reg),
11198             $src2$$constant & 0x1f);
11199   %}
11200 
11201   ins_pipe(ialu_reg_shift);
11202 %}
11203 
11204 // Shift Right Arithmetic Register
11205 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11206   match(Set dst (RShiftI src1 src2));
11207 
11208   ins_cost(INSN_COST * 2);
11209   format %{ "asrvw  $dst, $src1, $src2" %}
11210 
11211   ins_encode %{
11212     __ asrvw(as_Register($dst$$reg),
11213              as_Register($src1$$reg),
11214              as_Register($src2$$reg));
11215   %}
11216 
11217   ins_pipe(ialu_reg_reg_vshift);
11218 %}
11219 
11220 // Shift Right Arithmetic Immediate
11221 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11222   match(Set dst (RShiftI src1 src2));
11223 
11224   ins_cost(INSN_COST);
11225   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11226 
11227   ins_encode %{
11228     __ asrw(as_Register($dst$$reg),
11229             as_Register($src1$$reg),
11230             $src2$$constant & 0x1f);
11231   %}
11232 
11233   ins_pipe(ialu_reg_shift);
11234 %}
11235 
11236 // Combined Int Mask and Right Shift (using UBFM)
11237 // TODO
11238 
11239 // Long Shifts
11240 
11241 // Shift Left Register
11242 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11243   match(Set dst (LShiftL src1 src2));
11244 
11245   ins_cost(INSN_COST * 2);
11246   format %{ "lslv  $dst, $src1, $src2" %}
11247 
11248   ins_encode %{
11249     __ lslv(as_Register($dst$$reg),
11250             as_Register($src1$$reg),
11251             as_Register($src2$$reg));
11252   %}
11253 
11254   ins_pipe(ialu_reg_reg_vshift);
11255 %}
11256 
11257 // Shift Left Immediate
11258 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11259   match(Set dst (LShiftL src1 src2));
11260 
11261   ins_cost(INSN_COST);
11262   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11263 
11264   ins_encode %{
11265     __ lsl(as_Register($dst$$reg),
11266             as_Register($src1$$reg),
11267             $src2$$constant & 0x3f);
11268   %}
11269 
11270   ins_pipe(ialu_reg_shift);
11271 %}
11272 
11273 // Shift Right Logical Register
11274 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11275   match(Set dst (URShiftL src1 src2));
11276 
11277   ins_cost(INSN_COST * 2);
11278   format %{ "lsrv  $dst, $src1, $src2" %}
11279 
11280   ins_encode %{
11281     __ lsrv(as_Register($dst$$reg),
11282             as_Register($src1$$reg),
11283             as_Register($src2$$reg));
11284   %}
11285 
11286   ins_pipe(ialu_reg_reg_vshift);
11287 %}
11288 
11289 // Shift Right Logical Immediate
11290 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11291   match(Set dst (URShiftL src1 src2));
11292 
11293   ins_cost(INSN_COST);
11294   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11295 
11296   ins_encode %{
11297     __ lsr(as_Register($dst$$reg),
11298            as_Register($src1$$reg),
11299            $src2$$constant & 0x3f);
11300   %}
11301 
11302   ins_pipe(ialu_reg_shift);
11303 %}
11304 
11305 // A special-case pattern for card table stores.
11306 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11307   match(Set dst (URShiftL (CastP2X src1) src2));
11308 
11309   ins_cost(INSN_COST);
11310   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11311 
11312   ins_encode %{
11313     __ lsr(as_Register($dst$$reg),
11314            as_Register($src1$$reg),
11315            $src2$$constant & 0x3f);
11316   %}
11317 
11318   ins_pipe(ialu_reg_shift);
11319 %}
11320 
11321 // Shift Right Arithmetic Register
11322 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11323   match(Set dst (RShiftL src1 src2));
11324 
11325   ins_cost(INSN_COST * 2);
11326   format %{ "asrv  $dst, $src1, $src2" %}
11327 
11328   ins_encode %{
11329     __ asrv(as_Register($dst$$reg),
11330             as_Register($src1$$reg),
11331             as_Register($src2$$reg));
11332   %}
11333 
11334   ins_pipe(ialu_reg_reg_vshift);
11335 %}
11336 
11337 // Shift Right Arithmetic Immediate
11338 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11339   match(Set dst (RShiftL src1 src2));
11340 
11341   ins_cost(INSN_COST);
11342   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11343 
11344   ins_encode %{
11345     __ asr(as_Register($dst$$reg),
11346            as_Register($src1$$reg),
11347            $src2$$constant & 0x3f);
11348   %}
11349 
11350   ins_pipe(ialu_reg_shift);
11351 %}
11352 
11353 // BEGIN This section of the file is automatically generated. Do not edit --------------
11354 // This section is generated from aarch64_ad.m4
11355 
11356 
11357 // This pattern is automatically generated from aarch64_ad.m4
11358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11359 instruct regL_not_reg(iRegLNoSp dst,
11360                          iRegL src1, immL_M1 m1,
11361                          rFlagsReg cr) %{
11362   match(Set dst (XorL src1 m1));
11363   ins_cost(INSN_COST);
11364   format %{ "eon  $dst, $src1, zr" %}
11365 
11366   ins_encode %{
11367     __ eon(as_Register($dst$$reg),
11368               as_Register($src1$$reg),
11369               zr,
11370               Assembler::LSL, 0);
11371   %}
11372 
11373   ins_pipe(ialu_reg);
11374 %}
11375 
11376 // This pattern is automatically generated from aarch64_ad.m4
11377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11378 instruct regI_not_reg(iRegINoSp dst,
11379                          iRegIorL2I src1, immI_M1 m1,
11380                          rFlagsReg cr) %{
11381   match(Set dst (XorI src1 m1));
11382   ins_cost(INSN_COST);
11383   format %{ "eonw  $dst, $src1, zr" %}
11384 
11385   ins_encode %{
11386     __ eonw(as_Register($dst$$reg),
11387               as_Register($src1$$reg),
11388               zr,
11389               Assembler::LSL, 0);
11390   %}
11391 
11392   ins_pipe(ialu_reg);
11393 %}
11394 
11395 // This pattern is automatically generated from aarch64_ad.m4
11396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11397 instruct AndI_reg_not_reg(iRegINoSp dst,
11398                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11399   match(Set dst (AndI src1 (XorI src2 m1)));
11400   ins_cost(INSN_COST);
11401   format %{ "bicw  $dst, $src1, $src2" %}
11402 
11403   ins_encode %{
11404     __ bicw(as_Register($dst$$reg),
11405               as_Register($src1$$reg),
11406               as_Register($src2$$reg),
11407               Assembler::LSL, 0);
11408   %}
11409 
11410   ins_pipe(ialu_reg_reg);
11411 %}
11412 
11413 // This pattern is automatically generated from aarch64_ad.m4
11414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11415 instruct AndL_reg_not_reg(iRegLNoSp dst,
11416                          iRegL src1, iRegL src2, immL_M1 m1) %{
11417   match(Set dst (AndL src1 (XorL src2 m1)));
11418   ins_cost(INSN_COST);
11419   format %{ "bic  $dst, $src1, $src2" %}
11420 
11421   ins_encode %{
11422     __ bic(as_Register($dst$$reg),
11423               as_Register($src1$$reg),
11424               as_Register($src2$$reg),
11425               Assembler::LSL, 0);
11426   %}
11427 
11428   ins_pipe(ialu_reg_reg);
11429 %}
11430 
11431 // This pattern is automatically generated from aarch64_ad.m4
11432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11433 instruct OrI_reg_not_reg(iRegINoSp dst,
11434                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11435   match(Set dst (OrI src1 (XorI src2 m1)));
11436   ins_cost(INSN_COST);
11437   format %{ "ornw  $dst, $src1, $src2" %}
11438 
11439   ins_encode %{
11440     __ ornw(as_Register($dst$$reg),
11441               as_Register($src1$$reg),
11442               as_Register($src2$$reg),
11443               Assembler::LSL, 0);
11444   %}
11445 
11446   ins_pipe(ialu_reg_reg);
11447 %}
11448 
11449 // This pattern is automatically generated from aarch64_ad.m4
11450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11451 instruct OrL_reg_not_reg(iRegLNoSp dst,
11452                          iRegL src1, iRegL src2, immL_M1 m1) %{
11453   match(Set dst (OrL src1 (XorL src2 m1)));
11454   ins_cost(INSN_COST);
11455   format %{ "orn  $dst, $src1, $src2" %}
11456 
11457   ins_encode %{
11458     __ orn(as_Register($dst$$reg),
11459               as_Register($src1$$reg),
11460               as_Register($src2$$reg),
11461               Assembler::LSL, 0);
11462   %}
11463 
11464   ins_pipe(ialu_reg_reg);
11465 %}
11466 
11467 // This pattern is automatically generated from aarch64_ad.m4
11468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11469 instruct XorI_reg_not_reg(iRegINoSp dst,
11470                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11471   match(Set dst (XorI m1 (XorI src2 src1)));
11472   ins_cost(INSN_COST);
11473   format %{ "eonw  $dst, $src1, $src2" %}
11474 
11475   ins_encode %{
11476     __ eonw(as_Register($dst$$reg),
11477               as_Register($src1$$reg),
11478               as_Register($src2$$reg),
11479               Assembler::LSL, 0);
11480   %}
11481 
11482   ins_pipe(ialu_reg_reg);
11483 %}
11484 
11485 // This pattern is automatically generated from aarch64_ad.m4
11486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11487 instruct XorL_reg_not_reg(iRegLNoSp dst,
11488                          iRegL src1, iRegL src2, immL_M1 m1) %{
11489   match(Set dst (XorL m1 (XorL src2 src1)));
11490   ins_cost(INSN_COST);
11491   format %{ "eon  $dst, $src1, $src2" %}
11492 
11493   ins_encode %{
11494     __ eon(as_Register($dst$$reg),
11495               as_Register($src1$$reg),
11496               as_Register($src2$$reg),
11497               Assembler::LSL, 0);
11498   %}
11499 
11500   ins_pipe(ialu_reg_reg);
11501 %}
11502 
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505 // val & (-1 ^ (val >>> shift)) ==> bicw
11506 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11507                          iRegIorL2I src1, iRegIorL2I src2,
11508                          immI src3, immI_M1 src4) %{
11509   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11510   ins_cost(1.9 * INSN_COST);
11511   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11512 
11513   ins_encode %{
11514     __ bicw(as_Register($dst$$reg),
11515               as_Register($src1$$reg),
11516               as_Register($src2$$reg),
11517               Assembler::LSR,
11518               $src3$$constant & 0x1f);
11519   %}
11520 
11521   ins_pipe(ialu_reg_reg_shift);
11522 %}
11523 
11524 // This pattern is automatically generated from aarch64_ad.m4
11525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11526 // val & (-1 ^ (val >>> shift)) ==> bic
11527 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11528                          iRegL src1, iRegL src2,
11529                          immI src3, immL_M1 src4) %{
11530   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11531   ins_cost(1.9 * INSN_COST);
11532   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11533 
11534   ins_encode %{
11535     __ bic(as_Register($dst$$reg),
11536               as_Register($src1$$reg),
11537               as_Register($src2$$reg),
11538               Assembler::LSR,
11539               $src3$$constant & 0x3f);
11540   %}
11541 
11542   ins_pipe(ialu_reg_reg_shift);
11543 %}
11544 
11545 // This pattern is automatically generated from aarch64_ad.m4
11546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11547 // val & (-1 ^ (val >> shift)) ==> bicw
11548 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11549                          iRegIorL2I src1, iRegIorL2I src2,
11550                          immI src3, immI_M1 src4) %{
11551   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11552   ins_cost(1.9 * INSN_COST);
11553   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11554 
11555   ins_encode %{
11556     __ bicw(as_Register($dst$$reg),
11557               as_Register($src1$$reg),
11558               as_Register($src2$$reg),
11559               Assembler::ASR,
11560               $src3$$constant & 0x1f);
11561   %}
11562 
11563   ins_pipe(ialu_reg_reg_shift);
11564 %}
11565 
11566 // This pattern is automatically generated from aarch64_ad.m4
11567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11568 // val & (-1 ^ (val >> shift)) ==> bic
11569 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11570                          iRegL src1, iRegL src2,
11571                          immI src3, immL_M1 src4) %{
11572   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11573   ins_cost(1.9 * INSN_COST);
11574   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11575 
11576   ins_encode %{
11577     __ bic(as_Register($dst$$reg),
11578               as_Register($src1$$reg),
11579               as_Register($src2$$reg),
11580               Assembler::ASR,
11581               $src3$$constant & 0x3f);
11582   %}
11583 
11584   ins_pipe(ialu_reg_reg_shift);
11585 %}
11586 
11587 // This pattern is automatically generated from aarch64_ad.m4
11588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11589 // val & (-1 ^ (val ror shift)) ==> bicw
11590 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11591                          iRegIorL2I src1, iRegIorL2I src2,
11592                          immI src3, immI_M1 src4) %{
11593   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11594   ins_cost(1.9 * INSN_COST);
11595   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11596 
11597   ins_encode %{
11598     __ bicw(as_Register($dst$$reg),
11599               as_Register($src1$$reg),
11600               as_Register($src2$$reg),
11601               Assembler::ROR,
11602               $src3$$constant & 0x1f);
11603   %}
11604 
11605   ins_pipe(ialu_reg_reg_shift);
11606 %}
11607 
11608 // This pattern is automatically generated from aarch64_ad.m4
11609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11610 // val & (-1 ^ (val ror shift)) ==> bic
11611 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11612                          iRegL src1, iRegL src2,
11613                          immI src3, immL_M1 src4) %{
11614   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11615   ins_cost(1.9 * INSN_COST);
11616   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11617 
11618   ins_encode %{
11619     __ bic(as_Register($dst$$reg),
11620               as_Register($src1$$reg),
11621               as_Register($src2$$reg),
11622               Assembler::ROR,
11623               $src3$$constant & 0x3f);
11624   %}
11625 
11626   ins_pipe(ialu_reg_reg_shift);
11627 %}
11628 
11629 // This pattern is automatically generated from aarch64_ad.m4
11630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11631 // val & (-1 ^ (val << shift)) ==> bicw
11632 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11633                          iRegIorL2I src1, iRegIorL2I src2,
11634                          immI src3, immI_M1 src4) %{
11635   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11636   ins_cost(1.9 * INSN_COST);
11637   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11638 
11639   ins_encode %{
11640     __ bicw(as_Register($dst$$reg),
11641               as_Register($src1$$reg),
11642               as_Register($src2$$reg),
11643               Assembler::LSL,
11644               $src3$$constant & 0x1f);
11645   %}
11646 
11647   ins_pipe(ialu_reg_reg_shift);
11648 %}
11649 
11650 // This pattern is automatically generated from aarch64_ad.m4
11651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11652 // val & (-1 ^ (val << shift)) ==> bic
11653 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11654                          iRegL src1, iRegL src2,
11655                          immI src3, immL_M1 src4) %{
11656   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11657   ins_cost(1.9 * INSN_COST);
11658   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11659 
11660   ins_encode %{
11661     __ bic(as_Register($dst$$reg),
11662               as_Register($src1$$reg),
11663               as_Register($src2$$reg),
11664               Assembler::LSL,
11665               $src3$$constant & 0x3f);
11666   %}
11667 
11668   ins_pipe(ialu_reg_reg_shift);
11669 %}
11670 
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11674 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11675                          iRegIorL2I src1, iRegIorL2I src2,
11676                          immI src3, immI_M1 src4) %{
11677   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11678   ins_cost(1.9 * INSN_COST);
11679   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11680 
11681   ins_encode %{
11682     __ eonw(as_Register($dst$$reg),
11683               as_Register($src1$$reg),
11684               as_Register($src2$$reg),
11685               Assembler::LSR,
11686               $src3$$constant & 0x1f);
11687   %}
11688 
11689   ins_pipe(ialu_reg_reg_shift);
11690 %}
11691 
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 // val ^ (-1 ^ (val >>> shift)) ==> eon
11695 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11696                          iRegL src1, iRegL src2,
11697                          immI src3, immL_M1 src4) %{
11698   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11699   ins_cost(1.9 * INSN_COST);
11700   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11701 
11702   ins_encode %{
11703     __ eon(as_Register($dst$$reg),
11704               as_Register($src1$$reg),
11705               as_Register($src2$$reg),
11706               Assembler::LSR,
11707               $src3$$constant & 0x3f);
11708   %}
11709 
11710   ins_pipe(ialu_reg_reg_shift);
11711 %}
11712 
11713 // This pattern is automatically generated from aarch64_ad.m4
11714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11715 // val ^ (-1 ^ (val >> shift)) ==> eonw
11716 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11717                          iRegIorL2I src1, iRegIorL2I src2,
11718                          immI src3, immI_M1 src4) %{
11719   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11720   ins_cost(1.9 * INSN_COST);
11721   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11722 
11723   ins_encode %{
11724     __ eonw(as_Register($dst$$reg),
11725               as_Register($src1$$reg),
11726               as_Register($src2$$reg),
11727               Assembler::ASR,
11728               $src3$$constant & 0x1f);
11729   %}
11730 
11731   ins_pipe(ialu_reg_reg_shift);
11732 %}
11733 
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 // val ^ (-1 ^ (val >> shift)) ==> eon
11737 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11738                          iRegL src1, iRegL src2,
11739                          immI src3, immL_M1 src4) %{
11740   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11741   ins_cost(1.9 * INSN_COST);
11742   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11743 
11744   ins_encode %{
11745     __ eon(as_Register($dst$$reg),
11746               as_Register($src1$$reg),
11747               as_Register($src2$$reg),
11748               Assembler::ASR,
11749               $src3$$constant & 0x3f);
11750   %}
11751 
11752   ins_pipe(ialu_reg_reg_shift);
11753 %}
11754 
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 // val ^ (-1 ^ (val ror shift)) ==> eonw
11758 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11759                          iRegIorL2I src1, iRegIorL2I src2,
11760                          immI src3, immI_M1 src4) %{
11761   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11762   ins_cost(1.9 * INSN_COST);
11763   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11764 
11765   ins_encode %{
11766     __ eonw(as_Register($dst$$reg),
11767               as_Register($src1$$reg),
11768               as_Register($src2$$reg),
11769               Assembler::ROR,
11770               $src3$$constant & 0x1f);
11771   %}
11772 
11773   ins_pipe(ialu_reg_reg_shift);
11774 %}
11775 
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 // val ^ (-1 ^ (val ror shift)) ==> eon
11779 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11780                          iRegL src1, iRegL src2,
11781                          immI src3, immL_M1 src4) %{
11782   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11783   ins_cost(1.9 * INSN_COST);
11784   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11785 
11786   ins_encode %{
11787     __ eon(as_Register($dst$$reg),
11788               as_Register($src1$$reg),
11789               as_Register($src2$$reg),
11790               Assembler::ROR,
11791               $src3$$constant & 0x3f);
11792   %}
11793 
11794   ins_pipe(ialu_reg_reg_shift);
11795 %}
11796 
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 // val ^ (-1 ^ (val << shift)) ==> eonw
11800 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11801                          iRegIorL2I src1, iRegIorL2I src2,
11802                          immI src3, immI_M1 src4) %{
11803   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11804   ins_cost(1.9 * INSN_COST);
11805   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11806 
11807   ins_encode %{
11808     __ eonw(as_Register($dst$$reg),
11809               as_Register($src1$$reg),
11810               as_Register($src2$$reg),
11811               Assembler::LSL,
11812               $src3$$constant & 0x1f);
11813   %}
11814 
11815   ins_pipe(ialu_reg_reg_shift);
11816 %}
11817 
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 // val ^ (-1 ^ (val << shift)) ==> eon
11821 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11822                          iRegL src1, iRegL src2,
11823                          immI src3, immL_M1 src4) %{
11824   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11825   ins_cost(1.9 * INSN_COST);
11826   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11827 
11828   ins_encode %{
11829     __ eon(as_Register($dst$$reg),
11830               as_Register($src1$$reg),
11831               as_Register($src2$$reg),
11832               Assembler::LSL,
11833               $src3$$constant & 0x3f);
11834   %}
11835 
11836   ins_pipe(ialu_reg_reg_shift);
11837 %}
11838 
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 // val | (-1 ^ (val >>> shift)) ==> ornw
11842 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11843                          iRegIorL2I src1, iRegIorL2I src2,
11844                          immI src3, immI_M1 src4) %{
11845   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11846   ins_cost(1.9 * INSN_COST);
11847   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11848 
11849   ins_encode %{
11850     __ ornw(as_Register($dst$$reg),
11851               as_Register($src1$$reg),
11852               as_Register($src2$$reg),
11853               Assembler::LSR,
11854               $src3$$constant & 0x1f);
11855   %}
11856 
11857   ins_pipe(ialu_reg_reg_shift);
11858 %}
11859 
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 // val | (-1 ^ (val >>> shift)) ==> orn
11863 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11864                          iRegL src1, iRegL src2,
11865                          immI src3, immL_M1 src4) %{
11866   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11867   ins_cost(1.9 * INSN_COST);
11868   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11869 
11870   ins_encode %{
11871     __ orn(as_Register($dst$$reg),
11872               as_Register($src1$$reg),
11873               as_Register($src2$$reg),
11874               Assembler::LSR,
11875               $src3$$constant & 0x3f);
11876   %}
11877 
11878   ins_pipe(ialu_reg_reg_shift);
11879 %}
11880 
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 // val | (-1 ^ (val >> shift)) ==> ornw
11884 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11885                          iRegIorL2I src1, iRegIorL2I src2,
11886                          immI src3, immI_M1 src4) %{
11887   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11888   ins_cost(1.9 * INSN_COST);
11889   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11890 
11891   ins_encode %{
11892     __ ornw(as_Register($dst$$reg),
11893               as_Register($src1$$reg),
11894               as_Register($src2$$reg),
11895               Assembler::ASR,
11896               $src3$$constant & 0x1f);
11897   %}
11898 
11899   ins_pipe(ialu_reg_reg_shift);
11900 %}
11901 
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 // val | (-1 ^ (val >> shift)) ==> orn
11905 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11906                          iRegL src1, iRegL src2,
11907                          immI src3, immL_M1 src4) %{
11908   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11909   ins_cost(1.9 * INSN_COST);
11910   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11911 
11912   ins_encode %{
11913     __ orn(as_Register($dst$$reg),
11914               as_Register($src1$$reg),
11915               as_Register($src2$$reg),
11916               Assembler::ASR,
11917               $src3$$constant & 0x3f);
11918   %}
11919 
11920   ins_pipe(ialu_reg_reg_shift);
11921 %}
11922 
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 // val | (-1 ^ (val ror shift)) ==> ornw
11926 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11927                          iRegIorL2I src1, iRegIorL2I src2,
11928                          immI src3, immI_M1 src4) %{
11929   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11930   ins_cost(1.9 * INSN_COST);
11931   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11932 
11933   ins_encode %{
11934     __ ornw(as_Register($dst$$reg),
11935               as_Register($src1$$reg),
11936               as_Register($src2$$reg),
11937               Assembler::ROR,
11938               $src3$$constant & 0x1f);
11939   %}
11940 
11941   ins_pipe(ialu_reg_reg_shift);
11942 %}
11943 
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 // val | (-1 ^ (val ror shift)) ==> orn
11947 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11948                          iRegL src1, iRegL src2,
11949                          immI src3, immL_M1 src4) %{
11950   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11951   ins_cost(1.9 * INSN_COST);
11952   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11953 
11954   ins_encode %{
11955     __ orn(as_Register($dst$$reg),
11956               as_Register($src1$$reg),
11957               as_Register($src2$$reg),
11958               Assembler::ROR,
11959               $src3$$constant & 0x3f);
11960   %}
11961 
11962   ins_pipe(ialu_reg_reg_shift);
11963 %}
11964 
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 // val | (-1 ^ (val << shift)) ==> ornw
11968 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11969                          iRegIorL2I src1, iRegIorL2I src2,
11970                          immI src3, immI_M1 src4) %{
11971   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11972   ins_cost(1.9 * INSN_COST);
11973   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11974 
11975   ins_encode %{
11976     __ ornw(as_Register($dst$$reg),
11977               as_Register($src1$$reg),
11978               as_Register($src2$$reg),
11979               Assembler::LSL,
11980               $src3$$constant & 0x1f);
11981   %}
11982 
11983   ins_pipe(ialu_reg_reg_shift);
11984 %}
11985 
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 // val | (-1 ^ (val << shift)) ==> orn
11989 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11990                          iRegL src1, iRegL src2,
11991                          immI src3, immL_M1 src4) %{
11992   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11993   ins_cost(1.9 * INSN_COST);
11994   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11995 
11996   ins_encode %{
11997     __ orn(as_Register($dst$$reg),
11998               as_Register($src1$$reg),
11999               as_Register($src2$$reg),
12000               Assembler::LSL,
12001               $src3$$constant & 0x3f);
12002   %}
12003 
12004   ins_pipe(ialu_reg_reg_shift);
12005 %}
12006 
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12010                          iRegIorL2I src1, iRegIorL2I src2,
12011                          immI src3) %{
12012   match(Set dst (AndI src1 (URShiftI src2 src3)));
12013 
12014   ins_cost(1.9 * INSN_COST);
12015   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12016 
12017   ins_encode %{
12018     __ andw(as_Register($dst$$reg),
12019               as_Register($src1$$reg),
12020               as_Register($src2$$reg),
12021               Assembler::LSR,
12022               $src3$$constant & 0x1f);
12023   %}
12024 
12025   ins_pipe(ialu_reg_reg_shift);
12026 %}
12027 
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12031                          iRegL src1, iRegL src2,
12032                          immI src3) %{
12033   match(Set dst (AndL src1 (URShiftL src2 src3)));
12034 
12035   ins_cost(1.9 * INSN_COST);
12036   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12037 
12038   ins_encode %{
12039     __ andr(as_Register($dst$$reg),
12040               as_Register($src1$$reg),
12041               as_Register($src2$$reg),
12042               Assembler::LSR,
12043               $src3$$constant & 0x3f);
12044   %}
12045 
12046   ins_pipe(ialu_reg_reg_shift);
12047 %}
12048 
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12052                          iRegIorL2I src1, iRegIorL2I src2,
12053                          immI src3) %{
12054   match(Set dst (AndI src1 (RShiftI src2 src3)));
12055 
12056   ins_cost(1.9 * INSN_COST);
12057   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12058 
12059   ins_encode %{
12060     __ andw(as_Register($dst$$reg),
12061               as_Register($src1$$reg),
12062               as_Register($src2$$reg),
12063               Assembler::ASR,
12064               $src3$$constant & 0x1f);
12065   %}
12066 
12067   ins_pipe(ialu_reg_reg_shift);
12068 %}
12069 
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12073                          iRegL src1, iRegL src2,
12074                          immI src3) %{
12075   match(Set dst (AndL src1 (RShiftL src2 src3)));
12076 
12077   ins_cost(1.9 * INSN_COST);
12078   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12079 
12080   ins_encode %{
12081     __ andr(as_Register($dst$$reg),
12082               as_Register($src1$$reg),
12083               as_Register($src2$$reg),
12084               Assembler::ASR,
12085               $src3$$constant & 0x3f);
12086   %}
12087 
12088   ins_pipe(ialu_reg_reg_shift);
12089 %}
12090 
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12094                          iRegIorL2I src1, iRegIorL2I src2,
12095                          immI src3) %{
12096   match(Set dst (AndI src1 (LShiftI src2 src3)));
12097 
12098   ins_cost(1.9 * INSN_COST);
12099   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12100 
12101   ins_encode %{
12102     __ andw(as_Register($dst$$reg),
12103               as_Register($src1$$reg),
12104               as_Register($src2$$reg),
12105               Assembler::LSL,
12106               $src3$$constant & 0x1f);
12107   %}
12108 
12109   ins_pipe(ialu_reg_reg_shift);
12110 %}
12111 
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12115                          iRegL src1, iRegL src2,
12116                          immI src3) %{
12117   match(Set dst (AndL src1 (LShiftL src2 src3)));
12118 
12119   ins_cost(1.9 * INSN_COST);
12120   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12121 
12122   ins_encode %{
12123     __ andr(as_Register($dst$$reg),
12124               as_Register($src1$$reg),
12125               as_Register($src2$$reg),
12126               Assembler::LSL,
12127               $src3$$constant & 0x3f);
12128   %}
12129 
12130   ins_pipe(ialu_reg_reg_shift);
12131 %}
12132 
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12136                          iRegIorL2I src1, iRegIorL2I src2,
12137                          immI src3) %{
12138   match(Set dst (AndI src1 (RotateRight src2 src3)));
12139 
12140   ins_cost(1.9 * INSN_COST);
12141   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12142 
12143   ins_encode %{
12144     __ andw(as_Register($dst$$reg),
12145               as_Register($src1$$reg),
12146               as_Register($src2$$reg),
12147               Assembler::ROR,
12148               $src3$$constant & 0x1f);
12149   %}
12150 
12151   ins_pipe(ialu_reg_reg_shift);
12152 %}
12153 
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12157                          iRegL src1, iRegL src2,
12158                          immI src3) %{
12159   match(Set dst (AndL src1 (RotateRight src2 src3)));
12160 
12161   ins_cost(1.9 * INSN_COST);
12162   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12163 
12164   ins_encode %{
12165     __ andr(as_Register($dst$$reg),
12166               as_Register($src1$$reg),
12167               as_Register($src2$$reg),
12168               Assembler::ROR,
12169               $src3$$constant & 0x3f);
12170   %}
12171 
12172   ins_pipe(ialu_reg_reg_shift);
12173 %}
12174 
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12178                          iRegIorL2I src1, iRegIorL2I src2,
12179                          immI src3) %{
12180   match(Set dst (XorI src1 (URShiftI src2 src3)));
12181 
12182   ins_cost(1.9 * INSN_COST);
12183   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12184 
12185   ins_encode %{
12186     __ eorw(as_Register($dst$$reg),
12187               as_Register($src1$$reg),
12188               as_Register($src2$$reg),
12189               Assembler::LSR,
12190               $src3$$constant & 0x1f);
12191   %}
12192 
12193   ins_pipe(ialu_reg_reg_shift);
12194 %}
12195 
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12199                          iRegL src1, iRegL src2,
12200                          immI src3) %{
12201   match(Set dst (XorL src1 (URShiftL src2 src3)));
12202 
12203   ins_cost(1.9 * INSN_COST);
12204   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12205 
12206   ins_encode %{
12207     __ eor(as_Register($dst$$reg),
12208               as_Register($src1$$reg),
12209               as_Register($src2$$reg),
12210               Assembler::LSR,
12211               $src3$$constant & 0x3f);
12212   %}
12213 
12214   ins_pipe(ialu_reg_reg_shift);
12215 %}
12216 
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12220                          iRegIorL2I src1, iRegIorL2I src2,
12221                          immI src3) %{
12222   match(Set dst (XorI src1 (RShiftI src2 src3)));
12223 
12224   ins_cost(1.9 * INSN_COST);
12225   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12226 
12227   ins_encode %{
12228     __ eorw(as_Register($dst$$reg),
12229               as_Register($src1$$reg),
12230               as_Register($src2$$reg),
12231               Assembler::ASR,
12232               $src3$$constant & 0x1f);
12233   %}
12234 
12235   ins_pipe(ialu_reg_reg_shift);
12236 %}
12237 
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12241                          iRegL src1, iRegL src2,
12242                          immI src3) %{
12243   match(Set dst (XorL src1 (RShiftL src2 src3)));
12244 
12245   ins_cost(1.9 * INSN_COST);
12246   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12247 
12248   ins_encode %{
12249     __ eor(as_Register($dst$$reg),
12250               as_Register($src1$$reg),
12251               as_Register($src2$$reg),
12252               Assembler::ASR,
12253               $src3$$constant & 0x3f);
12254   %}
12255 
12256   ins_pipe(ialu_reg_reg_shift);
12257 %}
12258 
12259 // This pattern is automatically generated from aarch64_ad.m4
12260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12261 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12262                          iRegIorL2I src1, iRegIorL2I src2,
12263                          immI src3) %{
12264   match(Set dst (XorI src1 (LShiftI src2 src3)));
12265 
12266   ins_cost(1.9 * INSN_COST);
12267   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12268 
12269   ins_encode %{
12270     __ eorw(as_Register($dst$$reg),
12271               as_Register($src1$$reg),
12272               as_Register($src2$$reg),
12273               Assembler::LSL,
12274               $src3$$constant & 0x1f);
12275   %}
12276 
12277   ins_pipe(ialu_reg_reg_shift);
12278 %}
12279 
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12283                          iRegL src1, iRegL src2,
12284                          immI src3) %{
12285   match(Set dst (XorL src1 (LShiftL src2 src3)));
12286 
12287   ins_cost(1.9 * INSN_COST);
12288   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12289 
12290   ins_encode %{
12291     __ eor(as_Register($dst$$reg),
12292               as_Register($src1$$reg),
12293               as_Register($src2$$reg),
12294               Assembler::LSL,
12295               $src3$$constant & 0x3f);
12296   %}
12297 
12298   ins_pipe(ialu_reg_reg_shift);
12299 %}
12300 
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12304                          iRegIorL2I src1, iRegIorL2I src2,
12305                          immI src3) %{
12306   match(Set dst (XorI src1 (RotateRight src2 src3)));
12307 
12308   ins_cost(1.9 * INSN_COST);
12309   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12310 
12311   ins_encode %{
12312     __ eorw(as_Register($dst$$reg),
12313               as_Register($src1$$reg),
12314               as_Register($src2$$reg),
12315               Assembler::ROR,
12316               $src3$$constant & 0x1f);
12317   %}
12318 
12319   ins_pipe(ialu_reg_reg_shift);
12320 %}
12321 
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12325                          iRegL src1, iRegL src2,
12326                          immI src3) %{
12327   match(Set dst (XorL src1 (RotateRight src2 src3)));
12328 
12329   ins_cost(1.9 * INSN_COST);
12330   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12331 
12332   ins_encode %{
12333     __ eor(as_Register($dst$$reg),
12334               as_Register($src1$$reg),
12335               as_Register($src2$$reg),
12336               Assembler::ROR,
12337               $src3$$constant & 0x3f);
12338   %}
12339 
12340   ins_pipe(ialu_reg_reg_shift);
12341 %}
12342 
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12346                          iRegIorL2I src1, iRegIorL2I src2,
12347                          immI src3) %{
12348   match(Set dst (OrI src1 (URShiftI src2 src3)));
12349 
12350   ins_cost(1.9 * INSN_COST);
12351   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12352 
12353   ins_encode %{
12354     __ orrw(as_Register($dst$$reg),
12355               as_Register($src1$$reg),
12356               as_Register($src2$$reg),
12357               Assembler::LSR,
12358               $src3$$constant & 0x1f);
12359   %}
12360 
12361   ins_pipe(ialu_reg_reg_shift);
12362 %}
12363 
12364 // This pattern is automatically generated from aarch64_ad.m4
12365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12366 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12367                          iRegL src1, iRegL src2,
12368                          immI src3) %{
12369   match(Set dst (OrL src1 (URShiftL src2 src3)));
12370 
12371   ins_cost(1.9 * INSN_COST);
12372   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12373 
12374   ins_encode %{
12375     __ orr(as_Register($dst$$reg),
12376               as_Register($src1$$reg),
12377               as_Register($src2$$reg),
12378               Assembler::LSR,
12379               $src3$$constant & 0x3f);
12380   %}
12381 
12382   ins_pipe(ialu_reg_reg_shift);
12383 %}
12384 
12385 // This pattern is automatically generated from aarch64_ad.m4
12386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12387 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12388                          iRegIorL2I src1, iRegIorL2I src2,
12389                          immI src3) %{
12390   match(Set dst (OrI src1 (RShiftI src2 src3)));
12391 
12392   ins_cost(1.9 * INSN_COST);
12393   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12394 
12395   ins_encode %{
12396     __ orrw(as_Register($dst$$reg),
12397               as_Register($src1$$reg),
12398               as_Register($src2$$reg),
12399               Assembler::ASR,
12400               $src3$$constant & 0x1f);
12401   %}
12402 
12403   ins_pipe(ialu_reg_reg_shift);
12404 %}
12405 
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12409                          iRegL src1, iRegL src2,
12410                          immI src3) %{
12411   match(Set dst (OrL src1 (RShiftL src2 src3)));
12412 
12413   ins_cost(1.9 * INSN_COST);
12414   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12415 
12416   ins_encode %{
12417     __ orr(as_Register($dst$$reg),
12418               as_Register($src1$$reg),
12419               as_Register($src2$$reg),
12420               Assembler::ASR,
12421               $src3$$constant & 0x3f);
12422   %}
12423 
12424   ins_pipe(ialu_reg_reg_shift);
12425 %}
12426 
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12430                          iRegIorL2I src1, iRegIorL2I src2,
12431                          immI src3) %{
12432   match(Set dst (OrI src1 (LShiftI src2 src3)));
12433 
12434   ins_cost(1.9 * INSN_COST);
12435   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12436 
12437   ins_encode %{
12438     __ orrw(as_Register($dst$$reg),
12439               as_Register($src1$$reg),
12440               as_Register($src2$$reg),
12441               Assembler::LSL,
12442               $src3$$constant & 0x1f);
12443   %}
12444 
12445   ins_pipe(ialu_reg_reg_shift);
12446 %}
12447 
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12451                          iRegL src1, iRegL src2,
12452                          immI src3) %{
12453   match(Set dst (OrL src1 (LShiftL src2 src3)));
12454 
12455   ins_cost(1.9 * INSN_COST);
12456   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12457 
12458   ins_encode %{
12459     __ orr(as_Register($dst$$reg),
12460               as_Register($src1$$reg),
12461               as_Register($src2$$reg),
12462               Assembler::LSL,
12463               $src3$$constant & 0x3f);
12464   %}
12465 
12466   ins_pipe(ialu_reg_reg_shift);
12467 %}
12468 
12469 // This pattern is automatically generated from aarch64_ad.m4
12470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12471 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12472                          iRegIorL2I src1, iRegIorL2I src2,
12473                          immI src3) %{
12474   match(Set dst (OrI src1 (RotateRight src2 src3)));
12475 
12476   ins_cost(1.9 * INSN_COST);
12477   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12478 
12479   ins_encode %{
12480     __ orrw(as_Register($dst$$reg),
12481               as_Register($src1$$reg),
12482               as_Register($src2$$reg),
12483               Assembler::ROR,
12484               $src3$$constant & 0x1f);
12485   %}
12486 
12487   ins_pipe(ialu_reg_reg_shift);
12488 %}
12489 
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12493                          iRegL src1, iRegL src2,
12494                          immI src3) %{
12495   match(Set dst (OrL src1 (RotateRight src2 src3)));
12496 
12497   ins_cost(1.9 * INSN_COST);
12498   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12499 
12500   ins_encode %{
12501     __ orr(as_Register($dst$$reg),
12502               as_Register($src1$$reg),
12503               as_Register($src2$$reg),
12504               Assembler::ROR,
12505               $src3$$constant & 0x3f);
12506   %}
12507 
12508   ins_pipe(ialu_reg_reg_shift);
12509 %}
12510 
12511 // This pattern is automatically generated from aarch64_ad.m4
12512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12513 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12514                          iRegIorL2I src1, iRegIorL2I src2,
12515                          immI src3) %{
12516   match(Set dst (AddI src1 (URShiftI src2 src3)));
12517 
12518   ins_cost(1.9 * INSN_COST);
12519   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12520 
12521   ins_encode %{
12522     __ addw(as_Register($dst$$reg),
12523               as_Register($src1$$reg),
12524               as_Register($src2$$reg),
12525               Assembler::LSR,
12526               $src3$$constant & 0x1f);
12527   %}
12528 
12529   ins_pipe(ialu_reg_reg_shift);
12530 %}
12531 
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12535                          iRegL src1, iRegL src2,
12536                          immI src3) %{
12537   match(Set dst (AddL src1 (URShiftL src2 src3)));
12538 
12539   ins_cost(1.9 * INSN_COST);
12540   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12541 
12542   ins_encode %{
12543     __ add(as_Register($dst$$reg),
12544               as_Register($src1$$reg),
12545               as_Register($src2$$reg),
12546               Assembler::LSR,
12547               $src3$$constant & 0x3f);
12548   %}
12549 
12550   ins_pipe(ialu_reg_reg_shift);
12551 %}
12552 
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12556                          iRegIorL2I src1, iRegIorL2I src2,
12557                          immI src3) %{
12558   match(Set dst (AddI src1 (RShiftI src2 src3)));
12559 
12560   ins_cost(1.9 * INSN_COST);
12561   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12562 
12563   ins_encode %{
12564     __ addw(as_Register($dst$$reg),
12565               as_Register($src1$$reg),
12566               as_Register($src2$$reg),
12567               Assembler::ASR,
12568               $src3$$constant & 0x1f);
12569   %}
12570 
12571   ins_pipe(ialu_reg_reg_shift);
12572 %}
12573 
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12577                          iRegL src1, iRegL src2,
12578                          immI src3) %{
12579   match(Set dst (AddL src1 (RShiftL src2 src3)));
12580 
12581   ins_cost(1.9 * INSN_COST);
12582   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12583 
12584   ins_encode %{
12585     __ add(as_Register($dst$$reg),
12586               as_Register($src1$$reg),
12587               as_Register($src2$$reg),
12588               Assembler::ASR,
12589               $src3$$constant & 0x3f);
12590   %}
12591 
12592   ins_pipe(ialu_reg_reg_shift);
12593 %}
12594 
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12598                          iRegIorL2I src1, iRegIorL2I src2,
12599                          immI src3) %{
12600   match(Set dst (AddI src1 (LShiftI src2 src3)));
12601 
12602   ins_cost(1.9 * INSN_COST);
12603   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12604 
12605   ins_encode %{
12606     __ addw(as_Register($dst$$reg),
12607               as_Register($src1$$reg),
12608               as_Register($src2$$reg),
12609               Assembler::LSL,
12610               $src3$$constant & 0x1f);
12611   %}
12612 
12613   ins_pipe(ialu_reg_reg_shift);
12614 %}
12615 
12616 // This pattern is automatically generated from aarch64_ad.m4
12617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12618 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12619                          iRegL src1, iRegL src2,
12620                          immI src3) %{
12621   match(Set dst (AddL src1 (LShiftL src2 src3)));
12622 
12623   ins_cost(1.9 * INSN_COST);
12624   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12625 
12626   ins_encode %{
12627     __ add(as_Register($dst$$reg),
12628               as_Register($src1$$reg),
12629               as_Register($src2$$reg),
12630               Assembler::LSL,
12631               $src3$$constant & 0x3f);
12632   %}
12633 
12634   ins_pipe(ialu_reg_reg_shift);
12635 %}
12636 
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12640                          iRegIorL2I src1, iRegIorL2I src2,
12641                          immI src3) %{
12642   match(Set dst (SubI src1 (URShiftI src2 src3)));
12643 
12644   ins_cost(1.9 * INSN_COST);
12645   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12646 
12647   ins_encode %{
12648     __ subw(as_Register($dst$$reg),
12649               as_Register($src1$$reg),
12650               as_Register($src2$$reg),
12651               Assembler::LSR,
12652               $src3$$constant & 0x1f);
12653   %}
12654 
12655   ins_pipe(ialu_reg_reg_shift);
12656 %}
12657 
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12661                          iRegL src1, iRegL src2,
12662                          immI src3) %{
12663   match(Set dst (SubL src1 (URShiftL src2 src3)));
12664 
12665   ins_cost(1.9 * INSN_COST);
12666   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12667 
12668   ins_encode %{
12669     __ sub(as_Register($dst$$reg),
12670               as_Register($src1$$reg),
12671               as_Register($src2$$reg),
12672               Assembler::LSR,
12673               $src3$$constant & 0x3f);
12674   %}
12675 
12676   ins_pipe(ialu_reg_reg_shift);
12677 %}
12678 
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12682                          iRegIorL2I src1, iRegIorL2I src2,
12683                          immI src3) %{
12684   match(Set dst (SubI src1 (RShiftI src2 src3)));
12685 
12686   ins_cost(1.9 * INSN_COST);
12687   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12688 
12689   ins_encode %{
12690     __ subw(as_Register($dst$$reg),
12691               as_Register($src1$$reg),
12692               as_Register($src2$$reg),
12693               Assembler::ASR,
12694               $src3$$constant & 0x1f);
12695   %}
12696 
12697   ins_pipe(ialu_reg_reg_shift);
12698 %}
12699 
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12703                          iRegL src1, iRegL src2,
12704                          immI src3) %{
12705   match(Set dst (SubL src1 (RShiftL src2 src3)));
12706 
12707   ins_cost(1.9 * INSN_COST);
12708   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12709 
12710   ins_encode %{
12711     __ sub(as_Register($dst$$reg),
12712               as_Register($src1$$reg),
12713               as_Register($src2$$reg),
12714               Assembler::ASR,
12715               $src3$$constant & 0x3f);
12716   %}
12717 
12718   ins_pipe(ialu_reg_reg_shift);
12719 %}
12720 
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12724                          iRegIorL2I src1, iRegIorL2I src2,
12725                          immI src3) %{
12726   match(Set dst (SubI src1 (LShiftI src2 src3)));
12727 
12728   ins_cost(1.9 * INSN_COST);
12729   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12730 
12731   ins_encode %{
12732     __ subw(as_Register($dst$$reg),
12733               as_Register($src1$$reg),
12734               as_Register($src2$$reg),
12735               Assembler::LSL,
12736               $src3$$constant & 0x1f);
12737   %}
12738 
12739   ins_pipe(ialu_reg_reg_shift);
12740 %}
12741 
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12745                          iRegL src1, iRegL src2,
12746                          immI src3) %{
12747   match(Set dst (SubL src1 (LShiftL src2 src3)));
12748 
12749   ins_cost(1.9 * INSN_COST);
12750   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12751 
12752   ins_encode %{
12753     __ sub(as_Register($dst$$reg),
12754               as_Register($src1$$reg),
12755               as_Register($src2$$reg),
12756               Assembler::LSL,
12757               $src3$$constant & 0x3f);
12758   %}
12759 
12760   ins_pipe(ialu_reg_reg_shift);
12761 %}
12762 
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 
12766 // Shift Left followed by Shift Right.
12767 // This idiom is used by the compiler for the i2b bytecode etc.
12768 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12769 %{
12770   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12771   ins_cost(INSN_COST * 2);
12772   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12773   ins_encode %{
12774     int lshift = $lshift_count$$constant & 63;
12775     int rshift = $rshift_count$$constant & 63;
12776     int s = 63 - lshift;
12777     int r = (rshift - lshift) & 63;
12778     __ sbfm(as_Register($dst$$reg),
12779             as_Register($src$$reg),
12780             r, s);
12781   %}
12782 
12783   ins_pipe(ialu_reg_shift);
12784 %}
12785 
12786 // This pattern is automatically generated from aarch64_ad.m4
12787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12788 
12789 // Shift Left followed by Shift Right.
12790 // This idiom is used by the compiler for the i2b bytecode etc.
12791 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12792 %{
12793   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12794   ins_cost(INSN_COST * 2);
12795   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12796   ins_encode %{
12797     int lshift = $lshift_count$$constant & 31;
12798     int rshift = $rshift_count$$constant & 31;
12799     int s = 31 - lshift;
12800     int r = (rshift - lshift) & 31;
12801     __ sbfmw(as_Register($dst$$reg),
12802             as_Register($src$$reg),
12803             r, s);
12804   %}
12805 
12806   ins_pipe(ialu_reg_shift);
12807 %}
12808 
12809 // This pattern is automatically generated from aarch64_ad.m4
12810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12811 
12812 // Shift Left followed by Shift Right.
12813 // This idiom is used by the compiler for the i2b bytecode etc.
12814 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12815 %{
12816   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12817   ins_cost(INSN_COST * 2);
12818   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12819   ins_encode %{
12820     int lshift = $lshift_count$$constant & 63;
12821     int rshift = $rshift_count$$constant & 63;
12822     int s = 63 - lshift;
12823     int r = (rshift - lshift) & 63;
12824     __ ubfm(as_Register($dst$$reg),
12825             as_Register($src$$reg),
12826             r, s);
12827   %}
12828 
12829   ins_pipe(ialu_reg_shift);
12830 %}
12831 
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834 
12835 // Shift Left followed by Shift Right.
12836 // This idiom is used by the compiler for the i2b bytecode etc.
12837 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12838 %{
12839   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12840   ins_cost(INSN_COST * 2);
12841   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12842   ins_encode %{
12843     int lshift = $lshift_count$$constant & 31;
12844     int rshift = $rshift_count$$constant & 31;
12845     int s = 31 - lshift;
12846     int r = (rshift - lshift) & 31;
12847     __ ubfmw(as_Register($dst$$reg),
12848             as_Register($src$$reg),
12849             r, s);
12850   %}
12851 
12852   ins_pipe(ialu_reg_shift);
12853 %}
12854 
12855 // Bitfield extract with shift & mask
12856 
12857 // This pattern is automatically generated from aarch64_ad.m4
12858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12859 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12860 %{
12861   match(Set dst (AndI (URShiftI src rshift) mask));
12862   // Make sure we are not going to exceed what ubfxw can do.
12863   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12864 
12865   ins_cost(INSN_COST);
12866   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12867   ins_encode %{
12868     int rshift = $rshift$$constant & 31;
12869     intptr_t mask = $mask$$constant;
12870     int width = exact_log2(mask+1);
12871     __ ubfxw(as_Register($dst$$reg),
12872             as_Register($src$$reg), rshift, width);
12873   %}
12874   ins_pipe(ialu_reg_shift);
12875 %}
12876 
12877 // This pattern is automatically generated from aarch64_ad.m4
12878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12879 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12880 %{
12881   match(Set dst (AndL (URShiftL src rshift) mask));
12882   // Make sure we are not going to exceed what ubfx can do.
12883   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12884 
12885   ins_cost(INSN_COST);
12886   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12887   ins_encode %{
12888     int rshift = $rshift$$constant & 63;
12889     intptr_t mask = $mask$$constant;
12890     int width = exact_log2_long(mask+1);
12891     __ ubfx(as_Register($dst$$reg),
12892             as_Register($src$$reg), rshift, width);
12893   %}
12894   ins_pipe(ialu_reg_shift);
12895 %}
12896 
12897 
12898 // This pattern is automatically generated from aarch64_ad.m4
12899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12900 
12901 // We can use ubfx when extending an And with a mask when we know mask
12902 // is positive.  We know that because immI_bitmask guarantees it.
12903 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12904 %{
12905   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12906   // Make sure we are not going to exceed what ubfxw can do.
12907   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12908 
12909   ins_cost(INSN_COST * 2);
12910   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12911   ins_encode %{
12912     int rshift = $rshift$$constant & 31;
12913     intptr_t mask = $mask$$constant;
12914     int width = exact_log2(mask+1);
12915     __ ubfx(as_Register($dst$$reg),
12916             as_Register($src$$reg), rshift, width);
12917   %}
12918   ins_pipe(ialu_reg_shift);
12919 %}
12920 
12921 
12922 // This pattern is automatically generated from aarch64_ad.m4
12923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12924 
12925 // We can use ubfiz when masking by a positive number and then left shifting the result.
12926 // We know that the mask is positive because immI_bitmask guarantees it.
12927 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12928 %{
12929   match(Set dst (LShiftI (AndI src mask) lshift));
12930   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12931 
12932   ins_cost(INSN_COST);
12933   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12934   ins_encode %{
12935     int lshift = $lshift$$constant & 31;
12936     intptr_t mask = $mask$$constant;
12937     int width = exact_log2(mask+1);
12938     __ ubfizw(as_Register($dst$$reg),
12939           as_Register($src$$reg), lshift, width);
12940   %}
12941   ins_pipe(ialu_reg_shift);
12942 %}
12943 
12944 // This pattern is automatically generated from aarch64_ad.m4
12945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12946 
12947 // We can use ubfiz when masking by a positive number and then left shifting the result.
12948 // We know that the mask is positive because immL_bitmask guarantees it.
12949 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12950 %{
12951   match(Set dst (LShiftL (AndL src mask) lshift));
12952   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12953 
12954   ins_cost(INSN_COST);
12955   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12956   ins_encode %{
12957     int lshift = $lshift$$constant & 63;
12958     intptr_t mask = $mask$$constant;
12959     int width = exact_log2_long(mask+1);
12960     __ ubfiz(as_Register($dst$$reg),
12961           as_Register($src$$reg), lshift, width);
12962   %}
12963   ins_pipe(ialu_reg_shift);
12964 %}
12965 
12966 // This pattern is automatically generated from aarch64_ad.m4
12967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12968 
12969 // We can use ubfiz when masking by a positive number and then left shifting the result.
12970 // We know that the mask is positive because immI_bitmask guarantees it.
12971 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12972 %{
12973   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12974   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12975 
12976   ins_cost(INSN_COST);
12977   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12978   ins_encode %{
12979     int lshift = $lshift$$constant & 31;
12980     intptr_t mask = $mask$$constant;
12981     int width = exact_log2(mask+1);
12982     __ ubfizw(as_Register($dst$$reg),
12983           as_Register($src$$reg), lshift, width);
12984   %}
12985   ins_pipe(ialu_reg_shift);
12986 %}
12987 
12988 // This pattern is automatically generated from aarch64_ad.m4
12989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12990 
12991 // We can use ubfiz when masking by a positive number and then left shifting the result.
12992 // We know that the mask is positive because immL_bitmask guarantees it.
12993 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12994 %{
12995   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12996   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12997 
12998   ins_cost(INSN_COST);
12999   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13000   ins_encode %{
13001     int lshift = $lshift$$constant & 63;
13002     intptr_t mask = $mask$$constant;
13003     int width = exact_log2_long(mask+1);
13004     __ ubfiz(as_Register($dst$$reg),
13005           as_Register($src$$reg), lshift, width);
13006   %}
13007   ins_pipe(ialu_reg_shift);
13008 %}
13009 
13010 
13011 // This pattern is automatically generated from aarch64_ad.m4
13012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13013 
13014 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13015 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13016 %{
13017   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13018   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13019 
13020   ins_cost(INSN_COST);
13021   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13022   ins_encode %{
13023     int lshift = $lshift$$constant & 63;
13024     intptr_t mask = $mask$$constant;
13025     int width = exact_log2(mask+1);
13026     __ ubfiz(as_Register($dst$$reg),
13027              as_Register($src$$reg), lshift, width);
13028   %}
13029   ins_pipe(ialu_reg_shift);
13030 %}
13031 
13032 // This pattern is automatically generated from aarch64_ad.m4
13033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13034 
13035 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13036 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13037 %{
13038   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13039   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13040 
13041   ins_cost(INSN_COST);
13042   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13043   ins_encode %{
13044     int lshift = $lshift$$constant & 31;
13045     intptr_t mask = $mask$$constant;
13046     int width = exact_log2(mask+1);
13047     __ ubfiz(as_Register($dst$$reg),
13048              as_Register($src$$reg), lshift, width);
13049   %}
13050   ins_pipe(ialu_reg_shift);
13051 %}
13052 
13053 // This pattern is automatically generated from aarch64_ad.m4
13054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13055 
13056 // Can skip int2long conversions after AND with small bitmask
13057 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13058 %{
13059   match(Set dst (ConvI2L (AndI src msk)));
13060   ins_cost(INSN_COST);
13061   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13062   ins_encode %{
13063     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13064   %}
13065   ins_pipe(ialu_reg_shift);
13066 %}
13067 
13068 
13069 // Rotations
13070 // This pattern is automatically generated from aarch64_ad.m4
13071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13072 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13073 %{
13074   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13075   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13076 
13077   ins_cost(INSN_COST);
13078   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13079 
13080   ins_encode %{
13081     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13082             $rshift$$constant & 63);
13083   %}
13084   ins_pipe(ialu_reg_reg_extr);
13085 %}
13086 
13087 
13088 // This pattern is automatically generated from aarch64_ad.m4
13089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13090 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13091 %{
13092   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13093   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13094 
13095   ins_cost(INSN_COST);
13096   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13097 
13098   ins_encode %{
13099     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13100             $rshift$$constant & 31);
13101   %}
13102   ins_pipe(ialu_reg_reg_extr);
13103 %}
13104 
13105 
13106 // This pattern is automatically generated from aarch64_ad.m4
13107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13108 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13109 %{
13110   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13111   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13112 
13113   ins_cost(INSN_COST);
13114   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13115 
13116   ins_encode %{
13117     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13118             $rshift$$constant & 63);
13119   %}
13120   ins_pipe(ialu_reg_reg_extr);
13121 %}
13122 
13123 
13124 // This pattern is automatically generated from aarch64_ad.m4
13125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13126 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13127 %{
13128   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13129   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13130 
13131   ins_cost(INSN_COST);
13132   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13133 
13134   ins_encode %{
13135     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13136             $rshift$$constant & 31);
13137   %}
13138   ins_pipe(ialu_reg_reg_extr);
13139 %}
13140 
13141 
13142 // This pattern is automatically generated from aarch64_ad.m4
13143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13144 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13145 %{
13146   match(Set dst (RotateRight src shift));
13147 
13148   ins_cost(INSN_COST);
13149   format %{ "ror    $dst, $src, $shift" %}
13150 
13151   ins_encode %{
13152      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13153                $shift$$constant & 0x1f);
13154   %}
13155   ins_pipe(ialu_reg_reg_vshift);
13156 %}
13157 
13158 // This pattern is automatically generated from aarch64_ad.m4
13159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13160 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13161 %{
13162   match(Set dst (RotateRight src shift));
13163 
13164   ins_cost(INSN_COST);
13165   format %{ "ror    $dst, $src, $shift" %}
13166 
13167   ins_encode %{
13168      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13169                $shift$$constant & 0x3f);
13170   %}
13171   ins_pipe(ialu_reg_reg_vshift);
13172 %}
13173 
13174 // This pattern is automatically generated from aarch64_ad.m4
13175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13176 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13177 %{
13178   match(Set dst (RotateRight src shift));
13179 
13180   ins_cost(INSN_COST);
13181   format %{ "ror    $dst, $src, $shift" %}
13182 
13183   ins_encode %{
13184      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13185   %}
13186   ins_pipe(ialu_reg_reg_vshift);
13187 %}
13188 
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13192 %{
13193   match(Set dst (RotateRight src shift));
13194 
13195   ins_cost(INSN_COST);
13196   format %{ "ror    $dst, $src, $shift" %}
13197 
13198   ins_encode %{
13199      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13200   %}
13201   ins_pipe(ialu_reg_reg_vshift);
13202 %}
13203 
13204 // This pattern is automatically generated from aarch64_ad.m4
13205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13206 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13207 %{
13208   match(Set dst (RotateLeft src shift));
13209 
13210   ins_cost(INSN_COST);
13211   format %{ "rol    $dst, $src, $shift" %}
13212 
13213   ins_encode %{
13214      __ subw(rscratch1, zr, as_Register($shift$$reg));
13215      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13216   %}
13217   ins_pipe(ialu_reg_reg_vshift);
13218 %}
13219 
13220 // This pattern is automatically generated from aarch64_ad.m4
13221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13222 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13223 %{
13224   match(Set dst (RotateLeft src shift));
13225 
13226   ins_cost(INSN_COST);
13227   format %{ "rol    $dst, $src, $shift" %}
13228 
13229   ins_encode %{
13230      __ subw(rscratch1, zr, as_Register($shift$$reg));
13231      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13232   %}
13233   ins_pipe(ialu_reg_reg_vshift);
13234 %}
13235 
13236 
13237 // Add/subtract (extended)
13238 
13239 // This pattern is automatically generated from aarch64_ad.m4
13240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13241 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13242 %{
13243   match(Set dst (AddL src1 (ConvI2L src2)));
13244   ins_cost(INSN_COST);
13245   format %{ "add  $dst, $src1, $src2, sxtw" %}
13246 
13247    ins_encode %{
13248      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13249             as_Register($src2$$reg), ext::sxtw);
13250    %}
13251   ins_pipe(ialu_reg_reg);
13252 %}
13253 
13254 // This pattern is automatically generated from aarch64_ad.m4
13255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13256 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13257 %{
13258   match(Set dst (SubL src1 (ConvI2L src2)));
13259   ins_cost(INSN_COST);
13260   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13261 
13262    ins_encode %{
13263      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13264             as_Register($src2$$reg), ext::sxtw);
13265    %}
13266   ins_pipe(ialu_reg_reg);
13267 %}
13268 
13269 // This pattern is automatically generated from aarch64_ad.m4
13270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13271 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13272 %{
13273   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13274   ins_cost(INSN_COST);
13275   format %{ "add  $dst, $src1, $src2, sxth" %}
13276 
13277    ins_encode %{
13278      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13279             as_Register($src2$$reg), ext::sxth);
13280    %}
13281   ins_pipe(ialu_reg_reg);
13282 %}
13283 
13284 // This pattern is automatically generated from aarch64_ad.m4
13285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13286 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13287 %{
13288   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13289   ins_cost(INSN_COST);
13290   format %{ "add  $dst, $src1, $src2, sxtb" %}
13291 
13292    ins_encode %{
13293      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13294             as_Register($src2$$reg), ext::sxtb);
13295    %}
13296   ins_pipe(ialu_reg_reg);
13297 %}
13298 
13299 // This pattern is automatically generated from aarch64_ad.m4
13300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13301 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13302 %{
13303   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13304   ins_cost(INSN_COST);
13305   format %{ "add  $dst, $src1, $src2, uxtb" %}
13306 
13307    ins_encode %{
13308      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13309             as_Register($src2$$reg), ext::uxtb);
13310    %}
13311   ins_pipe(ialu_reg_reg);
13312 %}
13313 
13314 // This pattern is automatically generated from aarch64_ad.m4
13315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13316 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13317 %{
13318   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13319   ins_cost(INSN_COST);
13320   format %{ "add  $dst, $src1, $src2, sxth" %}
13321 
13322    ins_encode %{
13323      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13324             as_Register($src2$$reg), ext::sxth);
13325    %}
13326   ins_pipe(ialu_reg_reg);
13327 %}
13328 
13329 // This pattern is automatically generated from aarch64_ad.m4
13330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13331 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13332 %{
13333   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13334   ins_cost(INSN_COST);
13335   format %{ "add  $dst, $src1, $src2, sxtw" %}
13336 
13337    ins_encode %{
13338      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13339             as_Register($src2$$reg), ext::sxtw);
13340    %}
13341   ins_pipe(ialu_reg_reg);
13342 %}
13343 
13344 // This pattern is automatically generated from aarch64_ad.m4
13345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13346 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13347 %{
13348   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13349   ins_cost(INSN_COST);
13350   format %{ "add  $dst, $src1, $src2, sxtb" %}
13351 
13352    ins_encode %{
13353      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13354             as_Register($src2$$reg), ext::sxtb);
13355    %}
13356   ins_pipe(ialu_reg_reg);
13357 %}
13358 
13359 // This pattern is automatically generated from aarch64_ad.m4
13360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13361 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13362 %{
13363   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13364   ins_cost(INSN_COST);
13365   format %{ "add  $dst, $src1, $src2, uxtb" %}
13366 
13367    ins_encode %{
13368      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13369             as_Register($src2$$reg), ext::uxtb);
13370    %}
13371   ins_pipe(ialu_reg_reg);
13372 %}
13373 
13374 // This pattern is automatically generated from aarch64_ad.m4
13375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13376 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13377 %{
13378   match(Set dst (AddI src1 (AndI src2 mask)));
13379   ins_cost(INSN_COST);
13380   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13381 
13382    ins_encode %{
13383      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13384             as_Register($src2$$reg), ext::uxtb);
13385    %}
13386   ins_pipe(ialu_reg_reg);
13387 %}
13388 
13389 // This pattern is automatically generated from aarch64_ad.m4
13390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13391 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13392 %{
13393   match(Set dst (AddI src1 (AndI src2 mask)));
13394   ins_cost(INSN_COST);
13395   format %{ "addw  $dst, $src1, $src2, uxth" %}
13396 
13397    ins_encode %{
13398      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13399             as_Register($src2$$reg), ext::uxth);
13400    %}
13401   ins_pipe(ialu_reg_reg);
13402 %}
13403 
13404 // This pattern is automatically generated from aarch64_ad.m4
13405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13406 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13407 %{
13408   match(Set dst (AddL src1 (AndL src2 mask)));
13409   ins_cost(INSN_COST);
13410   format %{ "add  $dst, $src1, $src2, uxtb" %}
13411 
13412    ins_encode %{
13413      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13414             as_Register($src2$$reg), ext::uxtb);
13415    %}
13416   ins_pipe(ialu_reg_reg);
13417 %}
13418 
13419 // This pattern is automatically generated from aarch64_ad.m4
13420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13421 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13422 %{
13423   match(Set dst (AddL src1 (AndL src2 mask)));
13424   ins_cost(INSN_COST);
13425   format %{ "add  $dst, $src1, $src2, uxth" %}
13426 
13427    ins_encode %{
13428      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13429             as_Register($src2$$reg), ext::uxth);
13430    %}
13431   ins_pipe(ialu_reg_reg);
13432 %}
13433 
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13437 %{
13438   match(Set dst (AddL src1 (AndL src2 mask)));
13439   ins_cost(INSN_COST);
13440   format %{ "add  $dst, $src1, $src2, uxtw" %}
13441 
13442    ins_encode %{
13443      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13444             as_Register($src2$$reg), ext::uxtw);
13445    %}
13446   ins_pipe(ialu_reg_reg);
13447 %}
13448 
13449 // This pattern is automatically generated from aarch64_ad.m4
13450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13451 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13452 %{
13453   match(Set dst (SubI src1 (AndI src2 mask)));
13454   ins_cost(INSN_COST);
13455   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13456 
13457    ins_encode %{
13458      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13459             as_Register($src2$$reg), ext::uxtb);
13460    %}
13461   ins_pipe(ialu_reg_reg);
13462 %}
13463 
13464 // This pattern is automatically generated from aarch64_ad.m4
13465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13466 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13467 %{
13468   match(Set dst (SubI src1 (AndI src2 mask)));
13469   ins_cost(INSN_COST);
13470   format %{ "subw  $dst, $src1, $src2, uxth" %}
13471 
13472    ins_encode %{
13473      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13474             as_Register($src2$$reg), ext::uxth);
13475    %}
13476   ins_pipe(ialu_reg_reg);
13477 %}
13478 
13479 // This pattern is automatically generated from aarch64_ad.m4
13480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13481 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13482 %{
13483   match(Set dst (SubL src1 (AndL src2 mask)));
13484   ins_cost(INSN_COST);
13485   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13486 
13487    ins_encode %{
13488      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13489             as_Register($src2$$reg), ext::uxtb);
13490    %}
13491   ins_pipe(ialu_reg_reg);
13492 %}
13493 
13494 // This pattern is automatically generated from aarch64_ad.m4
13495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13496 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13497 %{
13498   match(Set dst (SubL src1 (AndL src2 mask)));
13499   ins_cost(INSN_COST);
13500   format %{ "sub  $dst, $src1, $src2, uxth" %}
13501 
13502    ins_encode %{
13503      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13504             as_Register($src2$$reg), ext::uxth);
13505    %}
13506   ins_pipe(ialu_reg_reg);
13507 %}
13508 
13509 // This pattern is automatically generated from aarch64_ad.m4
13510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13511 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13512 %{
13513   match(Set dst (SubL src1 (AndL src2 mask)));
13514   ins_cost(INSN_COST);
13515   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13516 
13517    ins_encode %{
13518      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13519             as_Register($src2$$reg), ext::uxtw);
13520    %}
13521   ins_pipe(ialu_reg_reg);
13522 %}
13523 
13524 
13525 // This pattern is automatically generated from aarch64_ad.m4
13526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13527 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13528 %{
13529   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13530   ins_cost(1.9 * INSN_COST);
13531   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13532 
13533    ins_encode %{
13534      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13535             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13536    %}
13537   ins_pipe(ialu_reg_reg_shift);
13538 %}
13539 
13540 // This pattern is automatically generated from aarch64_ad.m4
13541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13542 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13543 %{
13544   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13545   ins_cost(1.9 * INSN_COST);
13546   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13547 
13548    ins_encode %{
13549      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13550             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13551    %}
13552   ins_pipe(ialu_reg_reg_shift);
13553 %}
13554 
13555 // This pattern is automatically generated from aarch64_ad.m4
13556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13557 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13558 %{
13559   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13560   ins_cost(1.9 * INSN_COST);
13561   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13562 
13563    ins_encode %{
13564      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13565             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13566    %}
13567   ins_pipe(ialu_reg_reg_shift);
13568 %}
13569 
13570 // This pattern is automatically generated from aarch64_ad.m4
13571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13572 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13573 %{
13574   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13575   ins_cost(1.9 * INSN_COST);
13576   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13577 
13578    ins_encode %{
13579      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13580             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13581    %}
13582   ins_pipe(ialu_reg_reg_shift);
13583 %}
13584 
13585 // This pattern is automatically generated from aarch64_ad.m4
13586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13587 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13588 %{
13589   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13590   ins_cost(1.9 * INSN_COST);
13591   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13592 
13593    ins_encode %{
13594      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13595             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13596    %}
13597   ins_pipe(ialu_reg_reg_shift);
13598 %}
13599 
13600 // This pattern is automatically generated from aarch64_ad.m4
13601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13602 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13603 %{
13604   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13605   ins_cost(1.9 * INSN_COST);
13606   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13607 
13608    ins_encode %{
13609      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13610             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13611    %}
13612   ins_pipe(ialu_reg_reg_shift);
13613 %}
13614 
13615 // This pattern is automatically generated from aarch64_ad.m4
13616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13617 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13618 %{
13619   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13620   ins_cost(1.9 * INSN_COST);
13621   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13622 
13623    ins_encode %{
13624      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13625             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13626    %}
13627   ins_pipe(ialu_reg_reg_shift);
13628 %}
13629 
13630 // This pattern is automatically generated from aarch64_ad.m4
13631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13632 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13633 %{
13634   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13635   ins_cost(1.9 * INSN_COST);
13636   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13637 
13638    ins_encode %{
13639      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13640             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13641    %}
13642   ins_pipe(ialu_reg_reg_shift);
13643 %}
13644 
13645 // This pattern is automatically generated from aarch64_ad.m4
13646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13647 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13648 %{
13649   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13650   ins_cost(1.9 * INSN_COST);
13651   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13652 
13653    ins_encode %{
13654      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13655             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13656    %}
13657   ins_pipe(ialu_reg_reg_shift);
13658 %}
13659 
13660 // This pattern is automatically generated from aarch64_ad.m4
13661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13662 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13663 %{
13664   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13665   ins_cost(1.9 * INSN_COST);
13666   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13667 
13668    ins_encode %{
13669      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13670             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13671    %}
13672   ins_pipe(ialu_reg_reg_shift);
13673 %}
13674 
13675 // This pattern is automatically generated from aarch64_ad.m4
13676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13677 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13678 %{
13679   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13680   ins_cost(1.9 * INSN_COST);
13681   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13682 
13683    ins_encode %{
13684      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13685             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13686    %}
13687   ins_pipe(ialu_reg_reg_shift);
13688 %}
13689 
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13693 %{
13694   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13695   ins_cost(1.9 * INSN_COST);
13696   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13697 
13698    ins_encode %{
13699      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13700             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13701    %}
13702   ins_pipe(ialu_reg_reg_shift);
13703 %}
13704 
13705 // This pattern is automatically generated from aarch64_ad.m4
13706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13707 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13708 %{
13709   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13710   ins_cost(1.9 * INSN_COST);
13711   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13712 
13713    ins_encode %{
13714      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13715             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13716    %}
13717   ins_pipe(ialu_reg_reg_shift);
13718 %}
13719 
13720 // This pattern is automatically generated from aarch64_ad.m4
13721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13722 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13723 %{
13724   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13725   ins_cost(1.9 * INSN_COST);
13726   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13727 
13728    ins_encode %{
13729      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13730             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13731    %}
13732   ins_pipe(ialu_reg_reg_shift);
13733 %}
13734 
13735 // This pattern is automatically generated from aarch64_ad.m4
13736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13737 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13738 %{
13739   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13740   ins_cost(1.9 * INSN_COST);
13741   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13742 
13743    ins_encode %{
13744      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13745             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13746    %}
13747   ins_pipe(ialu_reg_reg_shift);
13748 %}
13749 
13750 // This pattern is automatically generated from aarch64_ad.m4
13751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13752 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13753 %{
13754   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13755   ins_cost(1.9 * INSN_COST);
13756   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13757 
13758    ins_encode %{
13759      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13760             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13761    %}
13762   ins_pipe(ialu_reg_reg_shift);
13763 %}
13764 
13765 // This pattern is automatically generated from aarch64_ad.m4
13766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13767 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13768 %{
13769   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13770   ins_cost(1.9 * INSN_COST);
13771   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13772 
13773    ins_encode %{
13774      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13775             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13776    %}
13777   ins_pipe(ialu_reg_reg_shift);
13778 %}
13779 
13780 // This pattern is automatically generated from aarch64_ad.m4
13781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13782 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13783 %{
13784   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13785   ins_cost(1.9 * INSN_COST);
13786   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13787 
13788    ins_encode %{
13789      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13790             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13791    %}
13792   ins_pipe(ialu_reg_reg_shift);
13793 %}
13794 
13795 // This pattern is automatically generated from aarch64_ad.m4
13796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13797 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13798 %{
13799   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13800   ins_cost(1.9 * INSN_COST);
13801   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13802 
13803    ins_encode %{
13804      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13805             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13806    %}
13807   ins_pipe(ialu_reg_reg_shift);
13808 %}
13809 
13810 // This pattern is automatically generated from aarch64_ad.m4
13811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13812 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13813 %{
13814   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13815   ins_cost(1.9 * INSN_COST);
13816   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13817 
13818    ins_encode %{
13819      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13820             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13821    %}
13822   ins_pipe(ialu_reg_reg_shift);
13823 %}
13824 
13825 // This pattern is automatically generated from aarch64_ad.m4
13826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13827 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13828 %{
13829   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13830   ins_cost(1.9 * INSN_COST);
13831   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13832 
13833    ins_encode %{
13834      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13835             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13836    %}
13837   ins_pipe(ialu_reg_reg_shift);
13838 %}
13839 
13840 // This pattern is automatically generated from aarch64_ad.m4
13841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13842 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13843 %{
13844   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13845   ins_cost(1.9 * INSN_COST);
13846   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13847 
13848    ins_encode %{
13849      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13850             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13851    %}
13852   ins_pipe(ialu_reg_reg_shift);
13853 %}
13854 
13855 
13856 
13857 // END This section of the file is automatically generated. Do not edit --------------
13858 
13859 
13860 // ============================================================================
13861 // Floating Point Arithmetic Instructions
13862 
13863 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13864   match(Set dst (AddF src1 src2));
13865 
13866   ins_cost(INSN_COST * 5);
13867   format %{ "fadds   $dst, $src1, $src2" %}
13868 
13869   ins_encode %{
13870     __ fadds(as_FloatRegister($dst$$reg),
13871              as_FloatRegister($src1$$reg),
13872              as_FloatRegister($src2$$reg));
13873   %}
13874 
13875   ins_pipe(fp_dop_reg_reg_s);
13876 %}
13877 
13878 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13879   match(Set dst (AddD src1 src2));
13880 
13881   ins_cost(INSN_COST * 5);
13882   format %{ "faddd   $dst, $src1, $src2" %}
13883 
13884   ins_encode %{
13885     __ faddd(as_FloatRegister($dst$$reg),
13886              as_FloatRegister($src1$$reg),
13887              as_FloatRegister($src2$$reg));
13888   %}
13889 
13890   ins_pipe(fp_dop_reg_reg_d);
13891 %}
13892 
13893 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13894   match(Set dst (SubF src1 src2));
13895 
13896   ins_cost(INSN_COST * 5);
13897   format %{ "fsubs   $dst, $src1, $src2" %}
13898 
13899   ins_encode %{
13900     __ fsubs(as_FloatRegister($dst$$reg),
13901              as_FloatRegister($src1$$reg),
13902              as_FloatRegister($src2$$reg));
13903   %}
13904 
13905   ins_pipe(fp_dop_reg_reg_s);
13906 %}
13907 
13908 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13909   match(Set dst (SubD src1 src2));
13910 
13911   ins_cost(INSN_COST * 5);
13912   format %{ "fsubd   $dst, $src1, $src2" %}
13913 
13914   ins_encode %{
13915     __ fsubd(as_FloatRegister($dst$$reg),
13916              as_FloatRegister($src1$$reg),
13917              as_FloatRegister($src2$$reg));
13918   %}
13919 
13920   ins_pipe(fp_dop_reg_reg_d);
13921 %}
13922 
13923 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13924   match(Set dst (MulF src1 src2));
13925 
13926   ins_cost(INSN_COST * 6);
13927   format %{ "fmuls   $dst, $src1, $src2" %}
13928 
13929   ins_encode %{
13930     __ fmuls(as_FloatRegister($dst$$reg),
13931              as_FloatRegister($src1$$reg),
13932              as_FloatRegister($src2$$reg));
13933   %}
13934 
13935   ins_pipe(fp_dop_reg_reg_s);
13936 %}
13937 
13938 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13939   match(Set dst (MulD src1 src2));
13940 
13941   ins_cost(INSN_COST * 6);
13942   format %{ "fmuld   $dst, $src1, $src2" %}
13943 
13944   ins_encode %{
13945     __ fmuld(as_FloatRegister($dst$$reg),
13946              as_FloatRegister($src1$$reg),
13947              as_FloatRegister($src2$$reg));
13948   %}
13949 
13950   ins_pipe(fp_dop_reg_reg_d);
13951 %}
13952 
13953 // src1 * src2 + src3
13954 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13955   predicate(UseFMA);
13956   match(Set dst (FmaF src3 (Binary src1 src2)));
13957 
13958   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13959 
13960   ins_encode %{
13961     __ fmadds(as_FloatRegister($dst$$reg),
13962              as_FloatRegister($src1$$reg),
13963              as_FloatRegister($src2$$reg),
13964              as_FloatRegister($src3$$reg));
13965   %}
13966 
13967   ins_pipe(pipe_class_default);
13968 %}
13969 
13970 // src1 * src2 + src3
13971 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13972   predicate(UseFMA);
13973   match(Set dst (FmaD src3 (Binary src1 src2)));
13974 
13975   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13976 
13977   ins_encode %{
13978     __ fmaddd(as_FloatRegister($dst$$reg),
13979              as_FloatRegister($src1$$reg),
13980              as_FloatRegister($src2$$reg),
13981              as_FloatRegister($src3$$reg));
13982   %}
13983 
13984   ins_pipe(pipe_class_default);
13985 %}
13986 
13987 // -src1 * src2 + src3
13988 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13989   predicate(UseFMA);
13990   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13991   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13992 
13993   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13994 
13995   ins_encode %{
13996     __ fmsubs(as_FloatRegister($dst$$reg),
13997               as_FloatRegister($src1$$reg),
13998               as_FloatRegister($src2$$reg),
13999               as_FloatRegister($src3$$reg));
14000   %}
14001 
14002   ins_pipe(pipe_class_default);
14003 %}
14004 
14005 // -src1 * src2 + src3
14006 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14007   predicate(UseFMA);
14008   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14009   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14010 
14011   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14012 
14013   ins_encode %{
14014     __ fmsubd(as_FloatRegister($dst$$reg),
14015               as_FloatRegister($src1$$reg),
14016               as_FloatRegister($src2$$reg),
14017               as_FloatRegister($src3$$reg));
14018   %}
14019 
14020   ins_pipe(pipe_class_default);
14021 %}
14022 
14023 // -src1 * src2 - src3
14024 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14025   predicate(UseFMA);
14026   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14027   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14028 
14029   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14030 
14031   ins_encode %{
14032     __ fnmadds(as_FloatRegister($dst$$reg),
14033                as_FloatRegister($src1$$reg),
14034                as_FloatRegister($src2$$reg),
14035                as_FloatRegister($src3$$reg));
14036   %}
14037 
14038   ins_pipe(pipe_class_default);
14039 %}
14040 
14041 // -src1 * src2 - src3
14042 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14043   predicate(UseFMA);
14044   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14045   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14046 
14047   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14048 
14049   ins_encode %{
14050     __ fnmaddd(as_FloatRegister($dst$$reg),
14051                as_FloatRegister($src1$$reg),
14052                as_FloatRegister($src2$$reg),
14053                as_FloatRegister($src3$$reg));
14054   %}
14055 
14056   ins_pipe(pipe_class_default);
14057 %}
14058 
14059 // src1 * src2 - src3
14060 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14061   predicate(UseFMA);
14062   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14063 
14064   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14065 
14066   ins_encode %{
14067     __ fnmsubs(as_FloatRegister($dst$$reg),
14068                as_FloatRegister($src1$$reg),
14069                as_FloatRegister($src2$$reg),
14070                as_FloatRegister($src3$$reg));
14071   %}
14072 
14073   ins_pipe(pipe_class_default);
14074 %}
14075 
14076 // src1 * src2 - src3
14077 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14078   predicate(UseFMA);
14079   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14080 
14081   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14082 
14083   ins_encode %{
14084   // n.b. insn name should be fnmsubd
14085     __ fnmsub(as_FloatRegister($dst$$reg),
14086               as_FloatRegister($src1$$reg),
14087               as_FloatRegister($src2$$reg),
14088               as_FloatRegister($src3$$reg));
14089   %}
14090 
14091   ins_pipe(pipe_class_default);
14092 %}
14093 
14094 
14095 // Math.max(FF)F
14096 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14097   match(Set dst (MaxF src1 src2));
14098 
14099   format %{ "fmaxs   $dst, $src1, $src2" %}
14100   ins_encode %{
14101     __ fmaxs(as_FloatRegister($dst$$reg),
14102              as_FloatRegister($src1$$reg),
14103              as_FloatRegister($src2$$reg));
14104   %}
14105 
14106   ins_pipe(fp_dop_reg_reg_s);
14107 %}
14108 
14109 // Math.min(FF)F
14110 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14111   match(Set dst (MinF src1 src2));
14112 
14113   format %{ "fmins   $dst, $src1, $src2" %}
14114   ins_encode %{
14115     __ fmins(as_FloatRegister($dst$$reg),
14116              as_FloatRegister($src1$$reg),
14117              as_FloatRegister($src2$$reg));
14118   %}
14119 
14120   ins_pipe(fp_dop_reg_reg_s);
14121 %}
14122 
14123 // Math.max(DD)D
14124 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14125   match(Set dst (MaxD src1 src2));
14126 
14127   format %{ "fmaxd   $dst, $src1, $src2" %}
14128   ins_encode %{
14129     __ fmaxd(as_FloatRegister($dst$$reg),
14130              as_FloatRegister($src1$$reg),
14131              as_FloatRegister($src2$$reg));
14132   %}
14133 
14134   ins_pipe(fp_dop_reg_reg_d);
14135 %}
14136 
14137 // Math.min(DD)D
14138 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14139   match(Set dst (MinD src1 src2));
14140 
14141   format %{ "fmind   $dst, $src1, $src2" %}
14142   ins_encode %{
14143     __ fmind(as_FloatRegister($dst$$reg),
14144              as_FloatRegister($src1$$reg),
14145              as_FloatRegister($src2$$reg));
14146   %}
14147 
14148   ins_pipe(fp_dop_reg_reg_d);
14149 %}
14150 
14151 
14152 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14153   match(Set dst (DivF src1  src2));
14154 
14155   ins_cost(INSN_COST * 18);
14156   format %{ "fdivs   $dst, $src1, $src2" %}
14157 
14158   ins_encode %{
14159     __ fdivs(as_FloatRegister($dst$$reg),
14160              as_FloatRegister($src1$$reg),
14161              as_FloatRegister($src2$$reg));
14162   %}
14163 
14164   ins_pipe(fp_div_s);
14165 %}
14166 
14167 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14168   match(Set dst (DivD src1  src2));
14169 
14170   ins_cost(INSN_COST * 32);
14171   format %{ "fdivd   $dst, $src1, $src2" %}
14172 
14173   ins_encode %{
14174     __ fdivd(as_FloatRegister($dst$$reg),
14175              as_FloatRegister($src1$$reg),
14176              as_FloatRegister($src2$$reg));
14177   %}
14178 
14179   ins_pipe(fp_div_d);
14180 %}
14181 
14182 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14183   match(Set dst (NegF src));
14184 
14185   ins_cost(INSN_COST * 3);
14186   format %{ "fneg   $dst, $src" %}
14187 
14188   ins_encode %{
14189     __ fnegs(as_FloatRegister($dst$$reg),
14190              as_FloatRegister($src$$reg));
14191   %}
14192 
14193   ins_pipe(fp_uop_s);
14194 %}
14195 
14196 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14197   match(Set dst (NegD src));
14198 
14199   ins_cost(INSN_COST * 3);
14200   format %{ "fnegd   $dst, $src" %}
14201 
14202   ins_encode %{
14203     __ fnegd(as_FloatRegister($dst$$reg),
14204              as_FloatRegister($src$$reg));
14205   %}
14206 
14207   ins_pipe(fp_uop_d);
14208 %}
14209 
14210 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14211 %{
14212   match(Set dst (AbsI src));
14213 
14214   effect(KILL cr);
14215   ins_cost(INSN_COST * 2);
14216   format %{ "cmpw  $src, zr\n\t"
14217             "cnegw $dst, $src, Assembler::LT\t# int abs"
14218   %}
14219 
14220   ins_encode %{
14221     __ cmpw(as_Register($src$$reg), zr);
14222     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14223   %}
14224   ins_pipe(pipe_class_default);
14225 %}
14226 
14227 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14228 %{
14229   match(Set dst (AbsL src));
14230 
14231   effect(KILL cr);
14232   ins_cost(INSN_COST * 2);
14233   format %{ "cmp  $src, zr\n\t"
14234             "cneg $dst, $src, Assembler::LT\t# long abs"
14235   %}
14236 
14237   ins_encode %{
14238     __ cmp(as_Register($src$$reg), zr);
14239     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14240   %}
14241   ins_pipe(pipe_class_default);
14242 %}
14243 
14244 instruct absF_reg(vRegF dst, vRegF src) %{
14245   match(Set dst (AbsF src));
14246 
14247   ins_cost(INSN_COST * 3);
14248   format %{ "fabss   $dst, $src" %}
14249   ins_encode %{
14250     __ fabss(as_FloatRegister($dst$$reg),
14251              as_FloatRegister($src$$reg));
14252   %}
14253 
14254   ins_pipe(fp_uop_s);
14255 %}
14256 
14257 instruct absD_reg(vRegD dst, vRegD src) %{
14258   match(Set dst (AbsD src));
14259 
14260   ins_cost(INSN_COST * 3);
14261   format %{ "fabsd   $dst, $src" %}
14262   ins_encode %{
14263     __ fabsd(as_FloatRegister($dst$$reg),
14264              as_FloatRegister($src$$reg));
14265   %}
14266 
14267   ins_pipe(fp_uop_d);
14268 %}
14269 
14270 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14271   match(Set dst (AbsF (SubF src1 src2)));
14272 
14273   ins_cost(INSN_COST * 3);
14274   format %{ "fabds   $dst, $src1, $src2" %}
14275   ins_encode %{
14276     __ fabds(as_FloatRegister($dst$$reg),
14277              as_FloatRegister($src1$$reg),
14278              as_FloatRegister($src2$$reg));
14279   %}
14280 
14281   ins_pipe(fp_uop_s);
14282 %}
14283 
14284 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14285   match(Set dst (AbsD (SubD src1 src2)));
14286 
14287   ins_cost(INSN_COST * 3);
14288   format %{ "fabdd   $dst, $src1, $src2" %}
14289   ins_encode %{
14290     __ fabdd(as_FloatRegister($dst$$reg),
14291              as_FloatRegister($src1$$reg),
14292              as_FloatRegister($src2$$reg));
14293   %}
14294 
14295   ins_pipe(fp_uop_d);
14296 %}
14297 
14298 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14299   match(Set dst (SqrtD src));
14300 
14301   ins_cost(INSN_COST * 50);
14302   format %{ "fsqrtd  $dst, $src" %}
14303   ins_encode %{
14304     __ fsqrtd(as_FloatRegister($dst$$reg),
14305              as_FloatRegister($src$$reg));
14306   %}
14307 
14308   ins_pipe(fp_div_s);
14309 %}
14310 
14311 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14312   match(Set dst (SqrtF src));
14313 
14314   ins_cost(INSN_COST * 50);
14315   format %{ "fsqrts  $dst, $src" %}
14316   ins_encode %{
14317     __ fsqrts(as_FloatRegister($dst$$reg),
14318              as_FloatRegister($src$$reg));
14319   %}
14320 
14321   ins_pipe(fp_div_d);
14322 %}
14323 
14324 // Math.rint, floor, ceil
14325 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14326   match(Set dst (RoundDoubleMode src rmode));
14327   format %{ "frint  $dst, $src, $rmode" %}
14328   ins_encode %{
14329     switch ($rmode$$constant) {
14330       case RoundDoubleModeNode::rmode_rint:
14331         __ frintnd(as_FloatRegister($dst$$reg),
14332                    as_FloatRegister($src$$reg));
14333         break;
14334       case RoundDoubleModeNode::rmode_floor:
14335         __ frintmd(as_FloatRegister($dst$$reg),
14336                    as_FloatRegister($src$$reg));
14337         break;
14338       case RoundDoubleModeNode::rmode_ceil:
14339         __ frintpd(as_FloatRegister($dst$$reg),
14340                    as_FloatRegister($src$$reg));
14341         break;
14342     }
14343   %}
14344   ins_pipe(fp_uop_d);
14345 %}
14346 
14347 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14348   match(Set dst (CopySignD src1 (Binary src2 zero)));
14349   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14350   format %{ "CopySignD  $dst $src1 $src2" %}
14351   ins_encode %{
14352     FloatRegister dst = as_FloatRegister($dst$$reg),
14353                   src1 = as_FloatRegister($src1$$reg),
14354                   src2 = as_FloatRegister($src2$$reg),
14355                   zero = as_FloatRegister($zero$$reg);
14356     __ fnegd(dst, zero);
14357     __ bsl(dst, __ T8B, src2, src1);
14358   %}
14359   ins_pipe(fp_uop_d);
14360 %}
14361 
14362 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14363   match(Set dst (CopySignF src1 src2));
14364   effect(TEMP_DEF dst, USE src1, USE src2);
14365   format %{ "CopySignF  $dst $src1 $src2" %}
14366   ins_encode %{
14367     FloatRegister dst = as_FloatRegister($dst$$reg),
14368                   src1 = as_FloatRegister($src1$$reg),
14369                   src2 = as_FloatRegister($src2$$reg);
14370     __ movi(dst, __ T2S, 0x80, 24);
14371     __ bsl(dst, __ T8B, src2, src1);
14372   %}
14373   ins_pipe(fp_uop_d);
14374 %}
14375 
14376 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14377   match(Set dst (SignumD src (Binary zero one)));
14378   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14379   format %{ "signumD  $dst, $src" %}
14380   ins_encode %{
14381     FloatRegister src = as_FloatRegister($src$$reg),
14382                   dst = as_FloatRegister($dst$$reg),
14383                   zero = as_FloatRegister($zero$$reg),
14384                   one = as_FloatRegister($one$$reg);
14385     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14386     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14387     // Bit selection instruction gets bit from "one" for each enabled bit in
14388     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14389     // NaN the whole "src" will be copied because "dst" is zero. For all other
14390     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14391     // from "src", and all other bits are copied from 1.0.
14392     __ bsl(dst, __ T8B, one, src);
14393   %}
14394   ins_pipe(fp_uop_d);
14395 %}
14396 
14397 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14398   match(Set dst (SignumF src (Binary zero one)));
14399   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14400   format %{ "signumF  $dst, $src" %}
14401   ins_encode %{
14402     FloatRegister src = as_FloatRegister($src$$reg),
14403                   dst = as_FloatRegister($dst$$reg),
14404                   zero = as_FloatRegister($zero$$reg),
14405                   one = as_FloatRegister($one$$reg);
14406     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14407     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14408     // Bit selection instruction gets bit from "one" for each enabled bit in
14409     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14410     // NaN the whole "src" will be copied because "dst" is zero. For all other
14411     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14412     // from "src", and all other bits are copied from 1.0.
14413     __ bsl(dst, __ T8B, one, src);
14414   %}
14415   ins_pipe(fp_uop_d);
14416 %}
14417 
14418 // ============================================================================
14419 // Logical Instructions
14420 
14421 // Integer Logical Instructions
14422 
14423 // And Instructions
14424 
14425 
14426 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14427   match(Set dst (AndI src1 src2));
14428 
14429   format %{ "andw  $dst, $src1, $src2\t# int" %}
14430 
14431   ins_cost(INSN_COST);
14432   ins_encode %{
14433     __ andw(as_Register($dst$$reg),
14434             as_Register($src1$$reg),
14435             as_Register($src2$$reg));
14436   %}
14437 
14438   ins_pipe(ialu_reg_reg);
14439 %}
14440 
14441 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14442   match(Set dst (AndI src1 src2));
14443 
14444   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14445 
14446   ins_cost(INSN_COST);
14447   ins_encode %{
14448     __ andw(as_Register($dst$$reg),
14449             as_Register($src1$$reg),
14450             (uint64_t)($src2$$constant));
14451   %}
14452 
14453   ins_pipe(ialu_reg_imm);
14454 %}
14455 
14456 // Or Instructions
14457 
14458 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14459   match(Set dst (OrI src1 src2));
14460 
14461   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14462 
14463   ins_cost(INSN_COST);
14464   ins_encode %{
14465     __ orrw(as_Register($dst$$reg),
14466             as_Register($src1$$reg),
14467             as_Register($src2$$reg));
14468   %}
14469 
14470   ins_pipe(ialu_reg_reg);
14471 %}
14472 
14473 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14474   match(Set dst (OrI src1 src2));
14475 
14476   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14477 
14478   ins_cost(INSN_COST);
14479   ins_encode %{
14480     __ orrw(as_Register($dst$$reg),
14481             as_Register($src1$$reg),
14482             (uint64_t)($src2$$constant));
14483   %}
14484 
14485   ins_pipe(ialu_reg_imm);
14486 %}
14487 
14488 // Xor Instructions
14489 
14490 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14491   match(Set dst (XorI src1 src2));
14492 
14493   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14494 
14495   ins_cost(INSN_COST);
14496   ins_encode %{
14497     __ eorw(as_Register($dst$$reg),
14498             as_Register($src1$$reg),
14499             as_Register($src2$$reg));
14500   %}
14501 
14502   ins_pipe(ialu_reg_reg);
14503 %}
14504 
14505 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14506   match(Set dst (XorI src1 src2));
14507 
14508   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14509 
14510   ins_cost(INSN_COST);
14511   ins_encode %{
14512     __ eorw(as_Register($dst$$reg),
14513             as_Register($src1$$reg),
14514             (uint64_t)($src2$$constant));
14515   %}
14516 
14517   ins_pipe(ialu_reg_imm);
14518 %}
14519 
14520 // Long Logical Instructions
14521 // TODO
14522 
14523 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14524   match(Set dst (AndL src1 src2));
14525 
14526   format %{ "and  $dst, $src1, $src2\t# int" %}
14527 
14528   ins_cost(INSN_COST);
14529   ins_encode %{
14530     __ andr(as_Register($dst$$reg),
14531             as_Register($src1$$reg),
14532             as_Register($src2$$reg));
14533   %}
14534 
14535   ins_pipe(ialu_reg_reg);
14536 %}
14537 
14538 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14539   match(Set dst (AndL src1 src2));
14540 
14541   format %{ "and  $dst, $src1, $src2\t# int" %}
14542 
14543   ins_cost(INSN_COST);
14544   ins_encode %{
14545     __ andr(as_Register($dst$$reg),
14546             as_Register($src1$$reg),
14547             (uint64_t)($src2$$constant));
14548   %}
14549 
14550   ins_pipe(ialu_reg_imm);
14551 %}
14552 
14553 // Or Instructions
14554 
14555 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14556   match(Set dst (OrL src1 src2));
14557 
14558   format %{ "orr  $dst, $src1, $src2\t# int" %}
14559 
14560   ins_cost(INSN_COST);
14561   ins_encode %{
14562     __ orr(as_Register($dst$$reg),
14563            as_Register($src1$$reg),
14564            as_Register($src2$$reg));
14565   %}
14566 
14567   ins_pipe(ialu_reg_reg);
14568 %}
14569 
14570 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14571   match(Set dst (OrL src1 src2));
14572 
14573   format %{ "orr  $dst, $src1, $src2\t# int" %}
14574 
14575   ins_cost(INSN_COST);
14576   ins_encode %{
14577     __ orr(as_Register($dst$$reg),
14578            as_Register($src1$$reg),
14579            (uint64_t)($src2$$constant));
14580   %}
14581 
14582   ins_pipe(ialu_reg_imm);
14583 %}
14584 
14585 // Xor Instructions
14586 
14587 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14588   match(Set dst (XorL src1 src2));
14589 
14590   format %{ "eor  $dst, $src1, $src2\t# int" %}
14591 
14592   ins_cost(INSN_COST);
14593   ins_encode %{
14594     __ eor(as_Register($dst$$reg),
14595            as_Register($src1$$reg),
14596            as_Register($src2$$reg));
14597   %}
14598 
14599   ins_pipe(ialu_reg_reg);
14600 %}
14601 
14602 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14603   match(Set dst (XorL src1 src2));
14604 
14605   ins_cost(INSN_COST);
14606   format %{ "eor  $dst, $src1, $src2\t# int" %}
14607 
14608   ins_encode %{
14609     __ eor(as_Register($dst$$reg),
14610            as_Register($src1$$reg),
14611            (uint64_t)($src2$$constant));
14612   %}
14613 
14614   ins_pipe(ialu_reg_imm);
14615 %}
14616 
14617 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14618 %{
14619   match(Set dst (ConvI2L src));
14620 
14621   ins_cost(INSN_COST);
14622   format %{ "sxtw  $dst, $src\t# i2l" %}
14623   ins_encode %{
14624     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14625   %}
14626   ins_pipe(ialu_reg_shift);
14627 %}
14628 
14629 // this pattern occurs in bigmath arithmetic
14630 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14631 %{
14632   match(Set dst (AndL (ConvI2L src) mask));
14633 
14634   ins_cost(INSN_COST);
14635   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14636   ins_encode %{
14637     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14638   %}
14639 
14640   ins_pipe(ialu_reg_shift);
14641 %}
14642 
14643 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14644   match(Set dst (ConvL2I src));
14645 
14646   ins_cost(INSN_COST);
14647   format %{ "movw  $dst, $src \t// l2i" %}
14648 
14649   ins_encode %{
14650     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14651   %}
14652 
14653   ins_pipe(ialu_reg);
14654 %}
14655 
14656 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14657 %{
14658   match(Set dst (Conv2B src));
14659   effect(KILL cr);
14660 
14661   format %{
14662     "cmpw $src, zr\n\t"
14663     "cset $dst, ne"
14664   %}
14665 
14666   ins_encode %{
14667     __ cmpw(as_Register($src$$reg), zr);
14668     __ cset(as_Register($dst$$reg), Assembler::NE);
14669   %}
14670 
14671   ins_pipe(ialu_reg);
14672 %}
14673 
14674 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14675 %{
14676   match(Set dst (Conv2B src));
14677   effect(KILL cr);
14678 
14679   format %{
14680     "cmp  $src, zr\n\t"
14681     "cset $dst, ne"
14682   %}
14683 
14684   ins_encode %{
14685     __ cmp(as_Register($src$$reg), zr);
14686     __ cset(as_Register($dst$$reg), Assembler::NE);
14687   %}
14688 
14689   ins_pipe(ialu_reg);
14690 %}
14691 
14692 instruct convD2F_reg(vRegF dst, vRegD src) %{
14693   match(Set dst (ConvD2F src));
14694 
14695   ins_cost(INSN_COST * 5);
14696   format %{ "fcvtd  $dst, $src \t// d2f" %}
14697 
14698   ins_encode %{
14699     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14700   %}
14701 
14702   ins_pipe(fp_d2f);
14703 %}
14704 
14705 instruct convF2D_reg(vRegD dst, vRegF src) %{
14706   match(Set dst (ConvF2D src));
14707 
14708   ins_cost(INSN_COST * 5);
14709   format %{ "fcvts  $dst, $src \t// f2d" %}
14710 
14711   ins_encode %{
14712     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14713   %}
14714 
14715   ins_pipe(fp_f2d);
14716 %}
14717 
14718 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14719   match(Set dst (ConvF2I src));
14720 
14721   ins_cost(INSN_COST * 5);
14722   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14723 
14724   ins_encode %{
14725     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14726   %}
14727 
14728   ins_pipe(fp_f2i);
14729 %}
14730 
14731 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14732   match(Set dst (ConvF2L src));
14733 
14734   ins_cost(INSN_COST * 5);
14735   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14736 
14737   ins_encode %{
14738     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14739   %}
14740 
14741   ins_pipe(fp_f2l);
14742 %}
14743 
14744 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14745   match(Set dst (ConvI2F src));
14746 
14747   ins_cost(INSN_COST * 5);
14748   format %{ "scvtfws  $dst, $src \t// i2f" %}
14749 
14750   ins_encode %{
14751     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14752   %}
14753 
14754   ins_pipe(fp_i2f);
14755 %}
14756 
14757 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14758   match(Set dst (ConvL2F src));
14759 
14760   ins_cost(INSN_COST * 5);
14761   format %{ "scvtfs  $dst, $src \t// l2f" %}
14762 
14763   ins_encode %{
14764     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14765   %}
14766 
14767   ins_pipe(fp_l2f);
14768 %}
14769 
14770 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14771   match(Set dst (ConvD2I src));
14772 
14773   ins_cost(INSN_COST * 5);
14774   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14775 
14776   ins_encode %{
14777     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14778   %}
14779 
14780   ins_pipe(fp_d2i);
14781 %}
14782 
14783 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14784   match(Set dst (ConvD2L src));
14785 
14786   ins_cost(INSN_COST * 5);
14787   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14788 
14789   ins_encode %{
14790     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14791   %}
14792 
14793   ins_pipe(fp_d2l);
14794 %}
14795 
14796 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14797   match(Set dst (ConvI2D src));
14798 
14799   ins_cost(INSN_COST * 5);
14800   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14801 
14802   ins_encode %{
14803     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14804   %}
14805 
14806   ins_pipe(fp_i2d);
14807 %}
14808 
14809 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14810   match(Set dst (ConvL2D src));
14811 
14812   ins_cost(INSN_COST * 5);
14813   format %{ "scvtfd  $dst, $src \t// l2d" %}
14814 
14815   ins_encode %{
14816     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14817   %}
14818 
14819   ins_pipe(fp_l2d);
14820 %}
14821 
14822 // stack <-> reg and reg <-> reg shuffles with no conversion
14823 
14824 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14825 
14826   match(Set dst (MoveF2I src));
14827 
14828   effect(DEF dst, USE src);
14829 
14830   ins_cost(4 * INSN_COST);
14831 
14832   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14833 
14834   ins_encode %{
14835     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14836   %}
14837 
14838   ins_pipe(iload_reg_reg);
14839 
14840 %}
14841 
14842 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14843 
14844   match(Set dst (MoveI2F src));
14845 
14846   effect(DEF dst, USE src);
14847 
14848   ins_cost(4 * INSN_COST);
14849 
14850   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14851 
14852   ins_encode %{
14853     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14854   %}
14855 
14856   ins_pipe(pipe_class_memory);
14857 
14858 %}
14859 
14860 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14861 
14862   match(Set dst (MoveD2L src));
14863 
14864   effect(DEF dst, USE src);
14865 
14866   ins_cost(4 * INSN_COST);
14867 
14868   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14869 
14870   ins_encode %{
14871     __ ldr($dst$$Register, Address(sp, $src$$disp));
14872   %}
14873 
14874   ins_pipe(iload_reg_reg);
14875 
14876 %}
14877 
14878 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14879 
14880   match(Set dst (MoveL2D src));
14881 
14882   effect(DEF dst, USE src);
14883 
14884   ins_cost(4 * INSN_COST);
14885 
14886   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14887 
14888   ins_encode %{
14889     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14890   %}
14891 
14892   ins_pipe(pipe_class_memory);
14893 
14894 %}
14895 
14896 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14897 
14898   match(Set dst (MoveF2I src));
14899 
14900   effect(DEF dst, USE src);
14901 
14902   ins_cost(INSN_COST);
14903 
14904   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14905 
14906   ins_encode %{
14907     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14908   %}
14909 
14910   ins_pipe(pipe_class_memory);
14911 
14912 %}
14913 
14914 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14915 
14916   match(Set dst (MoveI2F src));
14917 
14918   effect(DEF dst, USE src);
14919 
14920   ins_cost(INSN_COST);
14921 
14922   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14923 
14924   ins_encode %{
14925     __ strw($src$$Register, Address(sp, $dst$$disp));
14926   %}
14927 
14928   ins_pipe(istore_reg_reg);
14929 
14930 %}
14931 
14932 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14933 
14934   match(Set dst (MoveD2L src));
14935 
14936   effect(DEF dst, USE src);
14937 
14938   ins_cost(INSN_COST);
14939 
14940   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14941 
14942   ins_encode %{
14943     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14944   %}
14945 
14946   ins_pipe(pipe_class_memory);
14947 
14948 %}
14949 
14950 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14951 
14952   match(Set dst (MoveL2D src));
14953 
14954   effect(DEF dst, USE src);
14955 
14956   ins_cost(INSN_COST);
14957 
14958   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14959 
14960   ins_encode %{
14961     __ str($src$$Register, Address(sp, $dst$$disp));
14962   %}
14963 
14964   ins_pipe(istore_reg_reg);
14965 
14966 %}
14967 
14968 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14969 
14970   match(Set dst (MoveF2I src));
14971 
14972   effect(DEF dst, USE src);
14973 
14974   ins_cost(INSN_COST);
14975 
14976   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14977 
14978   ins_encode %{
14979     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14980   %}
14981 
14982   ins_pipe(fp_f2i);
14983 
14984 %}
14985 
14986 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14987 
14988   match(Set dst (MoveI2F src));
14989 
14990   effect(DEF dst, USE src);
14991 
14992   ins_cost(INSN_COST);
14993 
14994   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14995 
14996   ins_encode %{
14997     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14998   %}
14999 
15000   ins_pipe(fp_i2f);
15001 
15002 %}
15003 
15004 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15005 
15006   match(Set dst (MoveD2L src));
15007 
15008   effect(DEF dst, USE src);
15009 
15010   ins_cost(INSN_COST);
15011 
15012   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15013 
15014   ins_encode %{
15015     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15016   %}
15017 
15018   ins_pipe(fp_d2l);
15019 
15020 %}
15021 
15022 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15023 
15024   match(Set dst (MoveL2D src));
15025 
15026   effect(DEF dst, USE src);
15027 
15028   ins_cost(INSN_COST);
15029 
15030   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15031 
15032   ins_encode %{
15033     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15034   %}
15035 
15036   ins_pipe(fp_l2d);
15037 
15038 %}
15039 
15040 // ============================================================================
15041 // clearing of an array
15042 
15043 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15044 %{
15045   match(Set dummy (ClearArray cnt base));
15046   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15047 
15048   ins_cost(4 * INSN_COST);
15049   format %{ "ClearArray $cnt, $base" %}
15050 
15051   ins_encode %{
15052     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15053     if (tpc == NULL) {
15054       ciEnv::current()->record_failure("CodeCache is full");
15055       return;
15056     }
15057   %}
15058 
15059   ins_pipe(pipe_class_memory);
15060 %}
15061 
15062 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15063 %{
15064   predicate((uint64_t)n->in(2)->get_long()
15065             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15066   match(Set dummy (ClearArray cnt base));
15067   effect(TEMP temp, USE_KILL base, KILL cr);
15068 
15069   ins_cost(4 * INSN_COST);
15070   format %{ "ClearArray $cnt, $base" %}
15071 
15072   ins_encode %{
15073     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15074   %}
15075 
15076   ins_pipe(pipe_class_memory);
15077 %}
15078 
15079 // ============================================================================
15080 // Overflow Math Instructions
15081 
15082 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15083 %{
15084   match(Set cr (OverflowAddI op1 op2));
15085 
15086   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15087   ins_cost(INSN_COST);
15088   ins_encode %{
15089     __ cmnw($op1$$Register, $op2$$Register);
15090   %}
15091 
15092   ins_pipe(icmp_reg_reg);
15093 %}
15094 
15095 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15096 %{
15097   match(Set cr (OverflowAddI op1 op2));
15098 
15099   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15100   ins_cost(INSN_COST);
15101   ins_encode %{
15102     __ cmnw($op1$$Register, $op2$$constant);
15103   %}
15104 
15105   ins_pipe(icmp_reg_imm);
15106 %}
15107 
15108 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15109 %{
15110   match(Set cr (OverflowAddL op1 op2));
15111 
15112   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15113   ins_cost(INSN_COST);
15114   ins_encode %{
15115     __ cmn($op1$$Register, $op2$$Register);
15116   %}
15117 
15118   ins_pipe(icmp_reg_reg);
15119 %}
15120 
15121 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15122 %{
15123   match(Set cr (OverflowAddL op1 op2));
15124 
15125   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15126   ins_cost(INSN_COST);
15127   ins_encode %{
15128     __ cmn($op1$$Register, $op2$$constant);
15129   %}
15130 
15131   ins_pipe(icmp_reg_imm);
15132 %}
15133 
15134 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15135 %{
15136   match(Set cr (OverflowSubI op1 op2));
15137 
15138   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15139   ins_cost(INSN_COST);
15140   ins_encode %{
15141     __ cmpw($op1$$Register, $op2$$Register);
15142   %}
15143 
15144   ins_pipe(icmp_reg_reg);
15145 %}
15146 
15147 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15148 %{
15149   match(Set cr (OverflowSubI op1 op2));
15150 
15151   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15152   ins_cost(INSN_COST);
15153   ins_encode %{
15154     __ cmpw($op1$$Register, $op2$$constant);
15155   %}
15156 
15157   ins_pipe(icmp_reg_imm);
15158 %}
15159 
15160 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15161 %{
15162   match(Set cr (OverflowSubL op1 op2));
15163 
15164   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15165   ins_cost(INSN_COST);
15166   ins_encode %{
15167     __ cmp($op1$$Register, $op2$$Register);
15168   %}
15169 
15170   ins_pipe(icmp_reg_reg);
15171 %}
15172 
15173 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15174 %{
15175   match(Set cr (OverflowSubL op1 op2));
15176 
15177   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15178   ins_cost(INSN_COST);
15179   ins_encode %{
15180     __ subs(zr, $op1$$Register, $op2$$constant);
15181   %}
15182 
15183   ins_pipe(icmp_reg_imm);
15184 %}
15185 
15186 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15187 %{
15188   match(Set cr (OverflowSubI zero op1));
15189 
15190   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15191   ins_cost(INSN_COST);
15192   ins_encode %{
15193     __ cmpw(zr, $op1$$Register);
15194   %}
15195 
15196   ins_pipe(icmp_reg_imm);
15197 %}
15198 
15199 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15200 %{
15201   match(Set cr (OverflowSubL zero op1));
15202 
15203   format %{ "cmp   zr, $op1\t# overflow check long" %}
15204   ins_cost(INSN_COST);
15205   ins_encode %{
15206     __ cmp(zr, $op1$$Register);
15207   %}
15208 
15209   ins_pipe(icmp_reg_imm);
15210 %}
15211 
15212 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15213 %{
15214   match(Set cr (OverflowMulI op1 op2));
15215 
15216   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15217             "cmp   rscratch1, rscratch1, sxtw\n\t"
15218             "movw  rscratch1, #0x80000000\n\t"
15219             "cselw rscratch1, rscratch1, zr, NE\n\t"
15220             "cmpw  rscratch1, #1" %}
15221   ins_cost(5 * INSN_COST);
15222   ins_encode %{
15223     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15224     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15225     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15226     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15227     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15228   %}
15229 
15230   ins_pipe(pipe_slow);
15231 %}
15232 
15233 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15234 %{
15235   match(If cmp (OverflowMulI op1 op2));
15236   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15237             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15238   effect(USE labl, KILL cr);
15239 
15240   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15241             "cmp   rscratch1, rscratch1, sxtw\n\t"
15242             "b$cmp   $labl" %}
15243   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15244   ins_encode %{
15245     Label* L = $labl$$label;
15246     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15247     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15248     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15249     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15250   %}
15251 
15252   ins_pipe(pipe_serial);
15253 %}
15254 
15255 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15256 %{
15257   match(Set cr (OverflowMulL op1 op2));
15258 
15259   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15260             "smulh rscratch2, $op1, $op2\n\t"
15261             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15262             "movw  rscratch1, #0x80000000\n\t"
15263             "cselw rscratch1, rscratch1, zr, NE\n\t"
15264             "cmpw  rscratch1, #1" %}
15265   ins_cost(6 * INSN_COST);
15266   ins_encode %{
15267     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15268     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15269     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15270     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15271     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15272     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15273   %}
15274 
15275   ins_pipe(pipe_slow);
15276 %}
15277 
15278 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15279 %{
15280   match(If cmp (OverflowMulL op1 op2));
15281   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15282             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15283   effect(USE labl, KILL cr);
15284 
15285   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15286             "smulh rscratch2, $op1, $op2\n\t"
15287             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15288             "b$cmp $labl" %}
15289   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15290   ins_encode %{
15291     Label* L = $labl$$label;
15292     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15293     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15294     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15295     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15296     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15297   %}
15298 
15299   ins_pipe(pipe_serial);
15300 %}
15301 
15302 // ============================================================================
15303 // Compare Instructions
15304 
15305 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15306 %{
15307   match(Set cr (CmpI op1 op2));
15308 
15309   effect(DEF cr, USE op1, USE op2);
15310 
15311   ins_cost(INSN_COST);
15312   format %{ "cmpw  $op1, $op2" %}
15313 
15314   ins_encode(aarch64_enc_cmpw(op1, op2));
15315 
15316   ins_pipe(icmp_reg_reg);
15317 %}
15318 
15319 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15320 %{
15321   match(Set cr (CmpI op1 zero));
15322 
15323   effect(DEF cr, USE op1);
15324 
15325   ins_cost(INSN_COST);
15326   format %{ "cmpw $op1, 0" %}
15327 
15328   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15329 
15330   ins_pipe(icmp_reg_imm);
15331 %}
15332 
15333 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15334 %{
15335   match(Set cr (CmpI op1 op2));
15336 
15337   effect(DEF cr, USE op1);
15338 
15339   ins_cost(INSN_COST);
15340   format %{ "cmpw  $op1, $op2" %}
15341 
15342   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15343 
15344   ins_pipe(icmp_reg_imm);
15345 %}
15346 
15347 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15348 %{
15349   match(Set cr (CmpI op1 op2));
15350 
15351   effect(DEF cr, USE op1);
15352 
15353   ins_cost(INSN_COST * 2);
15354   format %{ "cmpw  $op1, $op2" %}
15355 
15356   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15357 
15358   ins_pipe(icmp_reg_imm);
15359 %}
15360 
15361 // Unsigned compare Instructions; really, same as signed compare
15362 // except it should only be used to feed an If or a CMovI which takes a
15363 // cmpOpU.
15364 
15365 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15366 %{
15367   match(Set cr (CmpU op1 op2));
15368 
15369   effect(DEF cr, USE op1, USE op2);
15370 
15371   ins_cost(INSN_COST);
15372   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15373 
15374   ins_encode(aarch64_enc_cmpw(op1, op2));
15375 
15376   ins_pipe(icmp_reg_reg);
15377 %}
15378 
15379 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15380 %{
15381   match(Set cr (CmpU op1 zero));
15382 
15383   effect(DEF cr, USE op1);
15384 
15385   ins_cost(INSN_COST);
15386   format %{ "cmpw $op1, #0\t# unsigned" %}
15387 
15388   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15389 
15390   ins_pipe(icmp_reg_imm);
15391 %}
15392 
15393 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15394 %{
15395   match(Set cr (CmpU op1 op2));
15396 
15397   effect(DEF cr, USE op1);
15398 
15399   ins_cost(INSN_COST);
15400   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15401 
15402   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15403 
15404   ins_pipe(icmp_reg_imm);
15405 %}
15406 
15407 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15408 %{
15409   match(Set cr (CmpU op1 op2));
15410 
15411   effect(DEF cr, USE op1);
15412 
15413   ins_cost(INSN_COST * 2);
15414   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15415 
15416   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15417 
15418   ins_pipe(icmp_reg_imm);
15419 %}
15420 
15421 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15422 %{
15423   match(Set cr (CmpL op1 op2));
15424 
15425   effect(DEF cr, USE op1, USE op2);
15426 
15427   ins_cost(INSN_COST);
15428   format %{ "cmp  $op1, $op2" %}
15429 
15430   ins_encode(aarch64_enc_cmp(op1, op2));
15431 
15432   ins_pipe(icmp_reg_reg);
15433 %}
15434 
15435 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15436 %{
15437   match(Set cr (CmpL op1 zero));
15438 
15439   effect(DEF cr, USE op1);
15440 
15441   ins_cost(INSN_COST);
15442   format %{ "tst  $op1" %}
15443 
15444   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15445 
15446   ins_pipe(icmp_reg_imm);
15447 %}
15448 
15449 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15450 %{
15451   match(Set cr (CmpL op1 op2));
15452 
15453   effect(DEF cr, USE op1);
15454 
15455   ins_cost(INSN_COST);
15456   format %{ "cmp  $op1, $op2" %}
15457 
15458   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15459 
15460   ins_pipe(icmp_reg_imm);
15461 %}
15462 
15463 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15464 %{
15465   match(Set cr (CmpL op1 op2));
15466 
15467   effect(DEF cr, USE op1);
15468 
15469   ins_cost(INSN_COST * 2);
15470   format %{ "cmp  $op1, $op2" %}
15471 
15472   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15473 
15474   ins_pipe(icmp_reg_imm);
15475 %}
15476 
15477 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15478 %{
15479   match(Set cr (CmpUL op1 op2));
15480 
15481   effect(DEF cr, USE op1, USE op2);
15482 
15483   ins_cost(INSN_COST);
15484   format %{ "cmp  $op1, $op2" %}
15485 
15486   ins_encode(aarch64_enc_cmp(op1, op2));
15487 
15488   ins_pipe(icmp_reg_reg);
15489 %}
15490 
15491 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15492 %{
15493   match(Set cr (CmpUL op1 zero));
15494 
15495   effect(DEF cr, USE op1);
15496 
15497   ins_cost(INSN_COST);
15498   format %{ "tst  $op1" %}
15499 
15500   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15501 
15502   ins_pipe(icmp_reg_imm);
15503 %}
15504 
15505 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15506 %{
15507   match(Set cr (CmpUL op1 op2));
15508 
15509   effect(DEF cr, USE op1);
15510 
15511   ins_cost(INSN_COST);
15512   format %{ "cmp  $op1, $op2" %}
15513 
15514   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15515 
15516   ins_pipe(icmp_reg_imm);
15517 %}
15518 
15519 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15520 %{
15521   match(Set cr (CmpUL op1 op2));
15522 
15523   effect(DEF cr, USE op1);
15524 
15525   ins_cost(INSN_COST * 2);
15526   format %{ "cmp  $op1, $op2" %}
15527 
15528   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15529 
15530   ins_pipe(icmp_reg_imm);
15531 %}
15532 
15533 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15534 %{
15535   match(Set cr (CmpP op1 op2));
15536 
15537   effect(DEF cr, USE op1, USE op2);
15538 
15539   ins_cost(INSN_COST);
15540   format %{ "cmp  $op1, $op2\t // ptr" %}
15541 
15542   ins_encode(aarch64_enc_cmpp(op1, op2));
15543 
15544   ins_pipe(icmp_reg_reg);
15545 %}
15546 
15547 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15548 %{
15549   match(Set cr (CmpN op1 op2));
15550 
15551   effect(DEF cr, USE op1, USE op2);
15552 
15553   ins_cost(INSN_COST);
15554   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15555 
15556   ins_encode(aarch64_enc_cmpn(op1, op2));
15557 
15558   ins_pipe(icmp_reg_reg);
15559 %}
15560 
15561 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15562 %{
15563   match(Set cr (CmpP op1 zero));
15564 
15565   effect(DEF cr, USE op1, USE zero);
15566 
15567   ins_cost(INSN_COST);
15568   format %{ "cmp  $op1, 0\t // ptr" %}
15569 
15570   ins_encode(aarch64_enc_testp(op1));
15571 
15572   ins_pipe(icmp_reg_imm);
15573 %}
15574 
15575 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15576 %{
15577   match(Set cr (CmpN op1 zero));
15578 
15579   effect(DEF cr, USE op1, USE zero);
15580 
15581   ins_cost(INSN_COST);
15582   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15583 
15584   ins_encode(aarch64_enc_testn(op1));
15585 
15586   ins_pipe(icmp_reg_imm);
15587 %}
15588 
15589 // FP comparisons
15590 //
15591 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15592 // using normal cmpOp. See declaration of rFlagsReg for details.
15593 
15594 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15595 %{
15596   match(Set cr (CmpF src1 src2));
15597 
15598   ins_cost(3 * INSN_COST);
15599   format %{ "fcmps $src1, $src2" %}
15600 
15601   ins_encode %{
15602     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15603   %}
15604 
15605   ins_pipe(pipe_class_compare);
15606 %}
15607 
15608 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15609 %{
15610   match(Set cr (CmpF src1 src2));
15611 
15612   ins_cost(3 * INSN_COST);
15613   format %{ "fcmps $src1, 0.0" %}
15614 
15615   ins_encode %{
15616     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15617   %}
15618 
15619   ins_pipe(pipe_class_compare);
15620 %}
15621 // FROM HERE
15622 
15623 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15624 %{
15625   match(Set cr (CmpD src1 src2));
15626 
15627   ins_cost(3 * INSN_COST);
15628   format %{ "fcmpd $src1, $src2" %}
15629 
15630   ins_encode %{
15631     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15632   %}
15633 
15634   ins_pipe(pipe_class_compare);
15635 %}
15636 
15637 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15638 %{
15639   match(Set cr (CmpD src1 src2));
15640 
15641   ins_cost(3 * INSN_COST);
15642   format %{ "fcmpd $src1, 0.0" %}
15643 
15644   ins_encode %{
15645     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15646   %}
15647 
15648   ins_pipe(pipe_class_compare);
15649 %}
15650 
15651 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15652 %{
15653   match(Set dst (CmpF3 src1 src2));
15654   effect(KILL cr);
15655 
15656   ins_cost(5 * INSN_COST);
15657   format %{ "fcmps $src1, $src2\n\t"
15658             "csinvw($dst, zr, zr, eq\n\t"
15659             "csnegw($dst, $dst, $dst, lt)"
15660   %}
15661 
15662   ins_encode %{
15663     Label done;
15664     FloatRegister s1 = as_FloatRegister($src1$$reg);
15665     FloatRegister s2 = as_FloatRegister($src2$$reg);
15666     Register d = as_Register($dst$$reg);
15667     __ fcmps(s1, s2);
15668     // installs 0 if EQ else -1
15669     __ csinvw(d, zr, zr, Assembler::EQ);
15670     // keeps -1 if less or unordered else installs 1
15671     __ csnegw(d, d, d, Assembler::LT);
15672     __ bind(done);
15673   %}
15674 
15675   ins_pipe(pipe_class_default);
15676 
15677 %}
15678 
15679 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15680 %{
15681   match(Set dst (CmpD3 src1 src2));
15682   effect(KILL cr);
15683 
15684   ins_cost(5 * INSN_COST);
15685   format %{ "fcmpd $src1, $src2\n\t"
15686             "csinvw($dst, zr, zr, eq\n\t"
15687             "csnegw($dst, $dst, $dst, lt)"
15688   %}
15689 
15690   ins_encode %{
15691     Label done;
15692     FloatRegister s1 = as_FloatRegister($src1$$reg);
15693     FloatRegister s2 = as_FloatRegister($src2$$reg);
15694     Register d = as_Register($dst$$reg);
15695     __ fcmpd(s1, s2);
15696     // installs 0 if EQ else -1
15697     __ csinvw(d, zr, zr, Assembler::EQ);
15698     // keeps -1 if less or unordered else installs 1
15699     __ csnegw(d, d, d, Assembler::LT);
15700     __ bind(done);
15701   %}
15702   ins_pipe(pipe_class_default);
15703 
15704 %}
15705 
15706 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15707 %{
15708   match(Set dst (CmpF3 src1 zero));
15709   effect(KILL cr);
15710 
15711   ins_cost(5 * INSN_COST);
15712   format %{ "fcmps $src1, 0.0\n\t"
15713             "csinvw($dst, zr, zr, eq\n\t"
15714             "csnegw($dst, $dst, $dst, lt)"
15715   %}
15716 
15717   ins_encode %{
15718     Label done;
15719     FloatRegister s1 = as_FloatRegister($src1$$reg);
15720     Register d = as_Register($dst$$reg);
15721     __ fcmps(s1, 0.0);
15722     // installs 0 if EQ else -1
15723     __ csinvw(d, zr, zr, Assembler::EQ);
15724     // keeps -1 if less or unordered else installs 1
15725     __ csnegw(d, d, d, Assembler::LT);
15726     __ bind(done);
15727   %}
15728 
15729   ins_pipe(pipe_class_default);
15730 
15731 %}
15732 
15733 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15734 %{
15735   match(Set dst (CmpD3 src1 zero));
15736   effect(KILL cr);
15737 
15738   ins_cost(5 * INSN_COST);
15739   format %{ "fcmpd $src1, 0.0\n\t"
15740             "csinvw($dst, zr, zr, eq\n\t"
15741             "csnegw($dst, $dst, $dst, lt)"
15742   %}
15743 
15744   ins_encode %{
15745     Label done;
15746     FloatRegister s1 = as_FloatRegister($src1$$reg);
15747     Register d = as_Register($dst$$reg);
15748     __ fcmpd(s1, 0.0);
15749     // installs 0 if EQ else -1
15750     __ csinvw(d, zr, zr, Assembler::EQ);
15751     // keeps -1 if less or unordered else installs 1
15752     __ csnegw(d, d, d, Assembler::LT);
15753     __ bind(done);
15754   %}
15755   ins_pipe(pipe_class_default);
15756 
15757 %}
15758 
15759 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15760 %{
15761   match(Set dst (CmpLTMask p q));
15762   effect(KILL cr);
15763 
15764   ins_cost(3 * INSN_COST);
15765 
15766   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15767             "csetw $dst, lt\n\t"
15768             "subw $dst, zr, $dst"
15769   %}
15770 
15771   ins_encode %{
15772     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15773     __ csetw(as_Register($dst$$reg), Assembler::LT);
15774     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15775   %}
15776 
15777   ins_pipe(ialu_reg_reg);
15778 %}
15779 
15780 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15781 %{
15782   match(Set dst (CmpLTMask src zero));
15783   effect(KILL cr);
15784 
15785   ins_cost(INSN_COST);
15786 
15787   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15788 
15789   ins_encode %{
15790     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15791   %}
15792 
15793   ins_pipe(ialu_reg_shift);
15794 %}
15795 
15796 // ============================================================================
15797 // Max and Min
15798 
15799 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15800 %{
15801   effect( DEF dst, USE src1, USE src2, USE cr );
15802 
15803   ins_cost(INSN_COST * 2);
15804   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15805 
15806   ins_encode %{
15807     __ cselw(as_Register($dst$$reg),
15808              as_Register($src1$$reg),
15809              as_Register($src2$$reg),
15810              Assembler::LT);
15811   %}
15812 
15813   ins_pipe(icond_reg_reg);
15814 %}
15815 
15816 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15817 %{
15818   match(Set dst (MinI src1 src2));
15819   ins_cost(INSN_COST * 3);
15820 
15821   expand %{
15822     rFlagsReg cr;
15823     compI_reg_reg(cr, src1, src2);
15824     cmovI_reg_reg_lt(dst, src1, src2, cr);
15825   %}
15826 
15827 %}
15828 // FROM HERE
15829 
15830 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15831 %{
15832   effect( DEF dst, USE src1, USE src2, USE cr );
15833 
15834   ins_cost(INSN_COST * 2);
15835   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15836 
15837   ins_encode %{
15838     __ cselw(as_Register($dst$$reg),
15839              as_Register($src1$$reg),
15840              as_Register($src2$$reg),
15841              Assembler::GT);
15842   %}
15843 
15844   ins_pipe(icond_reg_reg);
15845 %}
15846 
15847 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15848 %{
15849   match(Set dst (MaxI src1 src2));
15850   ins_cost(INSN_COST * 3);
15851   expand %{
15852     rFlagsReg cr;
15853     compI_reg_reg(cr, src1, src2);
15854     cmovI_reg_reg_gt(dst, src1, src2, cr);
15855   %}
15856 %}
15857 
15858 // ============================================================================
15859 // Branch Instructions
15860 
15861 // Direct Branch.
15862 instruct branch(label lbl)
15863 %{
15864   match(Goto);
15865 
15866   effect(USE lbl);
15867 
15868   ins_cost(BRANCH_COST);
15869   format %{ "b  $lbl" %}
15870 
15871   ins_encode(aarch64_enc_b(lbl));
15872 
15873   ins_pipe(pipe_branch);
15874 %}
15875 
15876 // Conditional Near Branch
15877 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15878 %{
15879   // Same match rule as `branchConFar'.
15880   match(If cmp cr);
15881 
15882   effect(USE lbl);
15883 
15884   ins_cost(BRANCH_COST);
15885   // If set to 1 this indicates that the current instruction is a
15886   // short variant of a long branch. This avoids using this
15887   // instruction in first-pass matching. It will then only be used in
15888   // the `Shorten_branches' pass.
15889   // ins_short_branch(1);
15890   format %{ "b$cmp  $lbl" %}
15891 
15892   ins_encode(aarch64_enc_br_con(cmp, lbl));
15893 
15894   ins_pipe(pipe_branch_cond);
15895 %}
15896 
15897 // Conditional Near Branch Unsigned
15898 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15899 %{
15900   // Same match rule as `branchConFar'.
15901   match(If cmp cr);
15902 
15903   effect(USE lbl);
15904 
15905   ins_cost(BRANCH_COST);
15906   // If set to 1 this indicates that the current instruction is a
15907   // short variant of a long branch. This avoids using this
15908   // instruction in first-pass matching. It will then only be used in
15909   // the `Shorten_branches' pass.
15910   // ins_short_branch(1);
15911   format %{ "b$cmp  $lbl\t# unsigned" %}
15912 
15913   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15914 
15915   ins_pipe(pipe_branch_cond);
15916 %}
15917 
15918 // Make use of CBZ and CBNZ.  These instructions, as well as being
15919 // shorter than (cmp; branch), have the additional benefit of not
15920 // killing the flags.
15921 
15922 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15923   match(If cmp (CmpI op1 op2));
15924   effect(USE labl);
15925 
15926   ins_cost(BRANCH_COST);
15927   format %{ "cbw$cmp   $op1, $labl" %}
15928   ins_encode %{
15929     Label* L = $labl$$label;
15930     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15931     if (cond == Assembler::EQ)
15932       __ cbzw($op1$$Register, *L);
15933     else
15934       __ cbnzw($op1$$Register, *L);
15935   %}
15936   ins_pipe(pipe_cmp_branch);
15937 %}
15938 
15939 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15940   match(If cmp (CmpL op1 op2));
15941   effect(USE labl);
15942 
15943   ins_cost(BRANCH_COST);
15944   format %{ "cb$cmp   $op1, $labl" %}
15945   ins_encode %{
15946     Label* L = $labl$$label;
15947     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15948     if (cond == Assembler::EQ)
15949       __ cbz($op1$$Register, *L);
15950     else
15951       __ cbnz($op1$$Register, *L);
15952   %}
15953   ins_pipe(pipe_cmp_branch);
15954 %}
15955 
15956 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15957   match(If cmp (CmpP op1 op2));
15958   effect(USE labl);
15959 
15960   ins_cost(BRANCH_COST);
15961   format %{ "cb$cmp   $op1, $labl" %}
15962   ins_encode %{
15963     Label* L = $labl$$label;
15964     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15965     if (cond == Assembler::EQ)
15966       __ cbz($op1$$Register, *L);
15967     else
15968       __ cbnz($op1$$Register, *L);
15969   %}
15970   ins_pipe(pipe_cmp_branch);
15971 %}
15972 
15973 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15974   match(If cmp (CmpN op1 op2));
15975   effect(USE labl);
15976 
15977   ins_cost(BRANCH_COST);
15978   format %{ "cbw$cmp   $op1, $labl" %}
15979   ins_encode %{
15980     Label* L = $labl$$label;
15981     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15982     if (cond == Assembler::EQ)
15983       __ cbzw($op1$$Register, *L);
15984     else
15985       __ cbnzw($op1$$Register, *L);
15986   %}
15987   ins_pipe(pipe_cmp_branch);
15988 %}
15989 
15990 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15991   match(If cmp (CmpP (DecodeN oop) zero));
15992   effect(USE labl);
15993 
15994   ins_cost(BRANCH_COST);
15995   format %{ "cb$cmp   $oop, $labl" %}
15996   ins_encode %{
15997     Label* L = $labl$$label;
15998     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15999     if (cond == Assembler::EQ)
16000       __ cbzw($oop$$Register, *L);
16001     else
16002       __ cbnzw($oop$$Register, *L);
16003   %}
16004   ins_pipe(pipe_cmp_branch);
16005 %}
16006 
16007 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16008   match(If cmp (CmpU op1 op2));
16009   effect(USE labl);
16010 
16011   ins_cost(BRANCH_COST);
16012   format %{ "cbw$cmp   $op1, $labl" %}
16013   ins_encode %{
16014     Label* L = $labl$$label;
16015     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16016     if (cond == Assembler::EQ || cond == Assembler::LS)
16017       __ cbzw($op1$$Register, *L);
16018     else
16019       __ cbnzw($op1$$Register, *L);
16020   %}
16021   ins_pipe(pipe_cmp_branch);
16022 %}
16023 
16024 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16025   match(If cmp (CmpUL op1 op2));
16026   effect(USE labl);
16027 
16028   ins_cost(BRANCH_COST);
16029   format %{ "cb$cmp   $op1, $labl" %}
16030   ins_encode %{
16031     Label* L = $labl$$label;
16032     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16033     if (cond == Assembler::EQ || cond == Assembler::LS)
16034       __ cbz($op1$$Register, *L);
16035     else
16036       __ cbnz($op1$$Register, *L);
16037   %}
16038   ins_pipe(pipe_cmp_branch);
16039 %}
16040 
16041 // Test bit and Branch
16042 
16043 // Patterns for short (< 32KiB) variants
16044 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16045   match(If cmp (CmpL op1 op2));
16046   effect(USE labl);
16047 
16048   ins_cost(BRANCH_COST);
16049   format %{ "cb$cmp   $op1, $labl # long" %}
16050   ins_encode %{
16051     Label* L = $labl$$label;
16052     Assembler::Condition cond =
16053       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16054     __ tbr(cond, $op1$$Register, 63, *L);
16055   %}
16056   ins_pipe(pipe_cmp_branch);
16057   ins_short_branch(1);
16058 %}
16059 
16060 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16061   match(If cmp (CmpI op1 op2));
16062   effect(USE labl);
16063 
16064   ins_cost(BRANCH_COST);
16065   format %{ "cb$cmp   $op1, $labl # int" %}
16066   ins_encode %{
16067     Label* L = $labl$$label;
16068     Assembler::Condition cond =
16069       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16070     __ tbr(cond, $op1$$Register, 31, *L);
16071   %}
16072   ins_pipe(pipe_cmp_branch);
16073   ins_short_branch(1);
16074 %}
16075 
16076 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16077   match(If cmp (CmpL (AndL op1 op2) op3));
16078   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16079   effect(USE labl);
16080 
16081   ins_cost(BRANCH_COST);
16082   format %{ "tb$cmp   $op1, $op2, $labl" %}
16083   ins_encode %{
16084     Label* L = $labl$$label;
16085     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16086     int bit = exact_log2_long($op2$$constant);
16087     __ tbr(cond, $op1$$Register, bit, *L);
16088   %}
16089   ins_pipe(pipe_cmp_branch);
16090   ins_short_branch(1);
16091 %}
16092 
16093 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16094   match(If cmp (CmpI (AndI op1 op2) op3));
16095   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16096   effect(USE labl);
16097 
16098   ins_cost(BRANCH_COST);
16099   format %{ "tb$cmp   $op1, $op2, $labl" %}
16100   ins_encode %{
16101     Label* L = $labl$$label;
16102     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16103     int bit = exact_log2((juint)$op2$$constant);
16104     __ tbr(cond, $op1$$Register, bit, *L);
16105   %}
16106   ins_pipe(pipe_cmp_branch);
16107   ins_short_branch(1);
16108 %}
16109 
16110 // And far variants
16111 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16112   match(If cmp (CmpL op1 op2));
16113   effect(USE labl);
16114 
16115   ins_cost(BRANCH_COST);
16116   format %{ "cb$cmp   $op1, $labl # long" %}
16117   ins_encode %{
16118     Label* L = $labl$$label;
16119     Assembler::Condition cond =
16120       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16121     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16122   %}
16123   ins_pipe(pipe_cmp_branch);
16124 %}
16125 
16126 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16127   match(If cmp (CmpI op1 op2));
16128   effect(USE labl);
16129 
16130   ins_cost(BRANCH_COST);
16131   format %{ "cb$cmp   $op1, $labl # int" %}
16132   ins_encode %{
16133     Label* L = $labl$$label;
16134     Assembler::Condition cond =
16135       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16136     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16137   %}
16138   ins_pipe(pipe_cmp_branch);
16139 %}
16140 
16141 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16142   match(If cmp (CmpL (AndL op1 op2) op3));
16143   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16144   effect(USE labl);
16145 
16146   ins_cost(BRANCH_COST);
16147   format %{ "tb$cmp   $op1, $op2, $labl" %}
16148   ins_encode %{
16149     Label* L = $labl$$label;
16150     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16151     int bit = exact_log2_long($op2$$constant);
16152     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16153   %}
16154   ins_pipe(pipe_cmp_branch);
16155 %}
16156 
16157 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16158   match(If cmp (CmpI (AndI op1 op2) op3));
16159   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16160   effect(USE labl);
16161 
16162   ins_cost(BRANCH_COST);
16163   format %{ "tb$cmp   $op1, $op2, $labl" %}
16164   ins_encode %{
16165     Label* L = $labl$$label;
16166     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16167     int bit = exact_log2((juint)$op2$$constant);
16168     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16169   %}
16170   ins_pipe(pipe_cmp_branch);
16171 %}
16172 
16173 // Test bits
16174 
16175 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16176   match(Set cr (CmpL (AndL op1 op2) op3));
16177   predicate(Assembler::operand_valid_for_logical_immediate
16178             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16179 
16180   ins_cost(INSN_COST);
16181   format %{ "tst $op1, $op2 # long" %}
16182   ins_encode %{
16183     __ tst($op1$$Register, $op2$$constant);
16184   %}
16185   ins_pipe(ialu_reg_reg);
16186 %}
16187 
16188 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16189   match(Set cr (CmpI (AndI op1 op2) op3));
16190   predicate(Assembler::operand_valid_for_logical_immediate
16191             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16192 
16193   ins_cost(INSN_COST);
16194   format %{ "tst $op1, $op2 # int" %}
16195   ins_encode %{
16196     __ tstw($op1$$Register, $op2$$constant);
16197   %}
16198   ins_pipe(ialu_reg_reg);
16199 %}
16200 
16201 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16202   match(Set cr (CmpL (AndL op1 op2) op3));
16203 
16204   ins_cost(INSN_COST);
16205   format %{ "tst $op1, $op2 # long" %}
16206   ins_encode %{
16207     __ tst($op1$$Register, $op2$$Register);
16208   %}
16209   ins_pipe(ialu_reg_reg);
16210 %}
16211 
16212 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16213   match(Set cr (CmpI (AndI op1 op2) op3));
16214 
16215   ins_cost(INSN_COST);
16216   format %{ "tstw $op1, $op2 # int" %}
16217   ins_encode %{
16218     __ tstw($op1$$Register, $op2$$Register);
16219   %}
16220   ins_pipe(ialu_reg_reg);
16221 %}
16222 
16223 
16224 // Conditional Far Branch
16225 // Conditional Far Branch Unsigned
16226 // TODO: fixme
16227 
16228 // counted loop end branch near
16229 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16230 %{
16231   match(CountedLoopEnd cmp cr);
16232 
16233   effect(USE lbl);
16234 
16235   ins_cost(BRANCH_COST);
16236   // short variant.
16237   // ins_short_branch(1);
16238   format %{ "b$cmp $lbl \t// counted loop end" %}
16239 
16240   ins_encode(aarch64_enc_br_con(cmp, lbl));
16241 
16242   ins_pipe(pipe_branch);
16243 %}
16244 
16245 // counted loop end branch near Unsigned
16246 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16247 %{
16248   match(CountedLoopEnd cmp cr);
16249 
16250   effect(USE lbl);
16251 
16252   ins_cost(BRANCH_COST);
16253   // short variant.
16254   // ins_short_branch(1);
16255   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16256 
16257   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16258 
16259   ins_pipe(pipe_branch);
16260 %}
16261 
16262 // counted loop end branch far
16263 // counted loop end branch far unsigned
16264 // TODO: fixme
16265 
16266 // ============================================================================
16267 // inlined locking and unlocking
16268 
16269 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16270 %{
16271   match(Set cr (FastLock object box));
16272   effect(TEMP tmp, TEMP tmp2);
16273 
16274   // TODO
16275   // identify correct cost
16276   ins_cost(5 * INSN_COST);
16277   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16278 
16279   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16280 
16281   ins_pipe(pipe_serial);
16282 %}
16283 
16284 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16285 %{
16286   match(Set cr (FastUnlock object box));
16287   effect(TEMP tmp, TEMP tmp2);
16288 
16289   ins_cost(5 * INSN_COST);
16290   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16291 
16292   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16293 
16294   ins_pipe(pipe_serial);
16295 %}
16296 
16297 
16298 // ============================================================================
16299 // Safepoint Instructions
16300 
16301 // TODO
16302 // provide a near and far version of this code
16303 
16304 instruct safePoint(rFlagsReg cr, iRegP poll)
16305 %{
16306   match(SafePoint poll);
16307   effect(KILL cr);
16308 
16309   format %{
16310     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16311   %}
16312   ins_encode %{
16313     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16314   %}
16315   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16316 %}
16317 
16318 
16319 // ============================================================================
16320 // Procedure Call/Return Instructions
16321 
16322 // Call Java Static Instruction
16323 
16324 instruct CallStaticJavaDirect(method meth)
16325 %{
16326   match(CallStaticJava);
16327 
16328   effect(USE meth);
16329 
16330   ins_cost(CALL_COST);
16331 
16332   format %{ "call,static $meth \t// ==> " %}
16333 
16334   ins_encode(aarch64_enc_java_static_call(meth),
16335              aarch64_enc_call_epilog);
16336 
16337   ins_pipe(pipe_class_call);
16338 %}
16339 
16340 // TO HERE
16341 
16342 // Call Java Dynamic Instruction
16343 instruct CallDynamicJavaDirect(method meth)
16344 %{
16345   match(CallDynamicJava);
16346 
16347   effect(USE meth);
16348 
16349   ins_cost(CALL_COST);
16350 
16351   format %{ "CALL,dynamic $meth \t// ==> " %}
16352 
16353   ins_encode(aarch64_enc_java_dynamic_call(meth),
16354              aarch64_enc_call_epilog);
16355 
16356   ins_pipe(pipe_class_call);
16357 %}
16358 
16359 // Call Runtime Instruction
16360 
16361 instruct CallRuntimeDirect(method meth)
16362 %{
16363   match(CallRuntime);
16364 
16365   effect(USE meth);
16366 
16367   ins_cost(CALL_COST);
16368 
16369   format %{ "CALL, runtime $meth" %}
16370 
16371   ins_encode( aarch64_enc_java_to_runtime(meth) );
16372 
16373   ins_pipe(pipe_class_call);
16374 %}
16375 
16376 // Call Runtime Instruction
16377 
16378 instruct CallLeafDirect(method meth)
16379 %{
16380   match(CallLeaf);
16381 
16382   effect(USE meth);
16383 
16384   ins_cost(CALL_COST);
16385 
16386   format %{ "CALL, runtime leaf $meth" %}
16387 
16388   ins_encode( aarch64_enc_java_to_runtime(meth) );
16389 
16390   ins_pipe(pipe_class_call);
16391 %}
16392 
16393 // Call Runtime Instruction
16394 
16395 instruct CallLeafNoFPDirect(method meth)
16396 %{
16397   match(CallLeafNoFP);
16398 
16399   effect(USE meth);
16400 
16401   ins_cost(CALL_COST);
16402 
16403   format %{ "CALL, runtime leaf nofp $meth" %}
16404 
16405   ins_encode( aarch64_enc_java_to_runtime(meth) );
16406 
16407   ins_pipe(pipe_class_call);
16408 %}
16409 
16410 instruct CallNativeDirect(method meth)
16411 %{
16412   match(CallNative);
16413 
16414   effect(USE meth);
16415 
16416   ins_cost(CALL_COST);
16417 
16418   format %{ "CALL, native $meth" %}
16419 
16420   ins_encode( aarch64_enc_java_to_runtime(meth) );
16421 
16422   ins_pipe(pipe_class_call);
16423 %}
16424 
16425 // Tail Call; Jump from runtime stub to Java code.
16426 // Also known as an 'interprocedural jump'.
16427 // Target of jump will eventually return to caller.
16428 // TailJump below removes the return address.
16429 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16430 %{
16431   match(TailCall jump_target method_ptr);
16432 
16433   ins_cost(CALL_COST);
16434 
16435   format %{ "br $jump_target\t# $method_ptr holds method" %}
16436 
16437   ins_encode(aarch64_enc_tail_call(jump_target));
16438 
16439   ins_pipe(pipe_class_call);
16440 %}
16441 
16442 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16443 %{
16444   match(TailJump jump_target ex_oop);
16445 
16446   ins_cost(CALL_COST);
16447 
16448   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16449 
16450   ins_encode(aarch64_enc_tail_jmp(jump_target));
16451 
16452   ins_pipe(pipe_class_call);
16453 %}
16454 
16455 // Create exception oop: created by stack-crawling runtime code.
16456 // Created exception is now available to this handler, and is setup
16457 // just prior to jumping to this handler. No code emitted.
16458 // TODO check
16459 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16460 instruct CreateException(iRegP_R0 ex_oop)
16461 %{
16462   match(Set ex_oop (CreateEx));
16463 
16464   format %{ " -- \t// exception oop; no code emitted" %}
16465 
16466   size(0);
16467 
16468   ins_encode( /*empty*/ );
16469 
16470   ins_pipe(pipe_class_empty);
16471 %}
16472 
16473 // Rethrow exception: The exception oop will come in the first
16474 // argument position. Then JUMP (not call) to the rethrow stub code.
16475 instruct RethrowException() %{
16476   match(Rethrow);
16477   ins_cost(CALL_COST);
16478 
16479   format %{ "b rethrow_stub" %}
16480 
16481   ins_encode( aarch64_enc_rethrow() );
16482 
16483   ins_pipe(pipe_class_call);
16484 %}
16485 
16486 
16487 // Return Instruction
16488 // epilog node loads ret address into lr as part of frame pop
16489 instruct Ret()
16490 %{
16491   match(Return);
16492 
16493   format %{ "ret\t// return register" %}
16494 
16495   ins_encode( aarch64_enc_ret() );
16496 
16497   ins_pipe(pipe_branch);
16498 %}
16499 
16500 // Die now.
16501 instruct ShouldNotReachHere() %{
16502   match(Halt);
16503 
16504   ins_cost(CALL_COST);
16505   format %{ "ShouldNotReachHere" %}
16506 
16507   ins_encode %{
16508     if (is_reachable()) {
16509       __ stop(_halt_reason);
16510     }
16511   %}
16512 
16513   ins_pipe(pipe_class_default);
16514 %}
16515 
16516 // ============================================================================
16517 // Partial Subtype Check
16518 //
16519 // superklass array for an instance of the superklass.  Set a hidden
16520 // internal cache on a hit (cache is checked with exposed code in
16521 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16522 // encoding ALSO sets flags.
16523 
16524 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16525 %{
16526   match(Set result (PartialSubtypeCheck sub super));
16527   effect(KILL cr, KILL temp);
16528 
16529   ins_cost(1100);  // slightly larger than the next version
16530   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16531 
16532   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16533 
16534   opcode(0x1); // Force zero of result reg on hit
16535 
16536   ins_pipe(pipe_class_memory);
16537 %}
16538 
16539 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16540 %{
16541   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16542   effect(KILL temp, KILL result);
16543 
16544   ins_cost(1100);  // slightly larger than the next version
16545   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16546 
16547   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16548 
16549   opcode(0x0); // Don't zero result reg on hit
16550 
16551   ins_pipe(pipe_class_memory);
16552 %}
16553 
16554 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16555                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16556 %{
16557   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16558   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16559   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16560 
16561   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16562   ins_encode %{
16563     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16564     __ string_compare($str1$$Register, $str2$$Register,
16565                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16566                       $tmp1$$Register, $tmp2$$Register,
16567                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16568   %}
16569   ins_pipe(pipe_class_memory);
16570 %}
16571 
16572 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16573                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16574 %{
16575   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16576   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16577   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16578 
16579   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16580   ins_encode %{
16581     __ string_compare($str1$$Register, $str2$$Register,
16582                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16583                       $tmp1$$Register, $tmp2$$Register,
16584                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16585   %}
16586   ins_pipe(pipe_class_memory);
16587 %}
16588 
16589 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16590                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16591                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16592 %{
16593   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16594   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16595   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16596          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16597 
16598   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16599   ins_encode %{
16600     __ string_compare($str1$$Register, $str2$$Register,
16601                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16602                       $tmp1$$Register, $tmp2$$Register,
16603                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16604                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16605   %}
16606   ins_pipe(pipe_class_memory);
16607 %}
16608 
16609 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16610                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16611                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16612 %{
16613   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16614   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16615   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16616          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16617 
16618   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16619   ins_encode %{
16620     __ string_compare($str1$$Register, $str2$$Register,
16621                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16622                       $tmp1$$Register, $tmp2$$Register,
16623                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16624                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16625   %}
16626   ins_pipe(pipe_class_memory);
16627 %}
16628 
16629 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16630        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16631        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16632 %{
16633   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16634   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16635   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16636          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16637   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16638 
16639   ins_encode %{
16640     __ string_indexof($str1$$Register, $str2$$Register,
16641                       $cnt1$$Register, $cnt2$$Register,
16642                       $tmp1$$Register, $tmp2$$Register,
16643                       $tmp3$$Register, $tmp4$$Register,
16644                       $tmp5$$Register, $tmp6$$Register,
16645                       -1, $result$$Register, StrIntrinsicNode::UU);
16646   %}
16647   ins_pipe(pipe_class_memory);
16648 %}
16649 
16650 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16651        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16652        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16653 %{
16654   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16655   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16656   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16657          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16658   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16659 
16660   ins_encode %{
16661     __ string_indexof($str1$$Register, $str2$$Register,
16662                       $cnt1$$Register, $cnt2$$Register,
16663                       $tmp1$$Register, $tmp2$$Register,
16664                       $tmp3$$Register, $tmp4$$Register,
16665                       $tmp5$$Register, $tmp6$$Register,
16666                       -1, $result$$Register, StrIntrinsicNode::LL);
16667   %}
16668   ins_pipe(pipe_class_memory);
16669 %}
16670 
16671 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16672        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16673        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16674 %{
16675   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16676   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16677   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16678          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16679   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16680 
16681   ins_encode %{
16682     __ string_indexof($str1$$Register, $str2$$Register,
16683                       $cnt1$$Register, $cnt2$$Register,
16684                       $tmp1$$Register, $tmp2$$Register,
16685                       $tmp3$$Register, $tmp4$$Register,
16686                       $tmp5$$Register, $tmp6$$Register,
16687                       -1, $result$$Register, StrIntrinsicNode::UL);
16688   %}
16689   ins_pipe(pipe_class_memory);
16690 %}
16691 
16692 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16693                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16694                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16695 %{
16696   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16697   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16698   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16699          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16700   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16701 
16702   ins_encode %{
16703     int icnt2 = (int)$int_cnt2$$constant;
16704     __ string_indexof($str1$$Register, $str2$$Register,
16705                       $cnt1$$Register, zr,
16706                       $tmp1$$Register, $tmp2$$Register,
16707                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16708                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16709   %}
16710   ins_pipe(pipe_class_memory);
16711 %}
16712 
16713 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16714                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16715                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16716 %{
16717   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16718   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16719   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16720          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16721   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16722 
16723   ins_encode %{
16724     int icnt2 = (int)$int_cnt2$$constant;
16725     __ string_indexof($str1$$Register, $str2$$Register,
16726                       $cnt1$$Register, zr,
16727                       $tmp1$$Register, $tmp2$$Register,
16728                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16729                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16730   %}
16731   ins_pipe(pipe_class_memory);
16732 %}
16733 
16734 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16735                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16736                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16737 %{
16738   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16739   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16740   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16741          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16742   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16743 
16744   ins_encode %{
16745     int icnt2 = (int)$int_cnt2$$constant;
16746     __ string_indexof($str1$$Register, $str2$$Register,
16747                       $cnt1$$Register, zr,
16748                       $tmp1$$Register, $tmp2$$Register,
16749                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16750                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16751   %}
16752   ins_pipe(pipe_class_memory);
16753 %}
16754 
16755 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16756                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16757                              iRegINoSp tmp3, rFlagsReg cr)
16758 %{
16759   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16760   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16761   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16762          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16763 
16764   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16765 
16766   ins_encode %{
16767     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16768                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16769                            $tmp3$$Register);
16770   %}
16771   ins_pipe(pipe_class_memory);
16772 %}
16773 
16774 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16775                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16776                               iRegINoSp tmp3, rFlagsReg cr)
16777 %{
16778   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16779   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16780   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16781          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16782 
16783   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16784 
16785   ins_encode %{
16786     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16787                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16788                             $tmp3$$Register);
16789   %}
16790   ins_pipe(pipe_class_memory);
16791 %}
16792 
16793 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16794                         iRegI_R0 result, rFlagsReg cr)
16795 %{
16796   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16797   match(Set result (StrEquals (Binary str1 str2) cnt));
16798   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16799 
16800   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16801   ins_encode %{
16802     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16803     __ string_equals($str1$$Register, $str2$$Register,
16804                      $result$$Register, $cnt$$Register, 1);
16805   %}
16806   ins_pipe(pipe_class_memory);
16807 %}
16808 
16809 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16810                         iRegI_R0 result, rFlagsReg cr)
16811 %{
16812   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16813   match(Set result (StrEquals (Binary str1 str2) cnt));
16814   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16815 
16816   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16817   ins_encode %{
16818     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16819     __ string_equals($str1$$Register, $str2$$Register,
16820                      $result$$Register, $cnt$$Register, 2);
16821   %}
16822   ins_pipe(pipe_class_memory);
16823 %}
16824 
16825 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16826                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16827                        iRegP_R10 tmp, rFlagsReg cr)
16828 %{
16829   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16830   match(Set result (AryEq ary1 ary2));
16831   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16832 
16833   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16834   ins_encode %{
16835     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16836                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16837                                    $result$$Register, $tmp$$Register, 1);
16838     if (tpc == NULL) {
16839       ciEnv::current()->record_failure("CodeCache is full");
16840       return;
16841     }
16842   %}
16843   ins_pipe(pipe_class_memory);
16844 %}
16845 
16846 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16847                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16848                        iRegP_R10 tmp, rFlagsReg cr)
16849 %{
16850   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16851   match(Set result (AryEq ary1 ary2));
16852   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16853 
16854   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16855   ins_encode %{
16856     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16857                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16858                                    $result$$Register, $tmp$$Register, 2);
16859     if (tpc == NULL) {
16860       ciEnv::current()->record_failure("CodeCache is full");
16861       return;
16862     }
16863   %}
16864   ins_pipe(pipe_class_memory);
16865 %}
16866 
16867 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16868 %{
16869   match(Set result (HasNegatives ary1 len));
16870   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16871   format %{ "has negatives byte[] $ary1,$len -> $result" %}
16872   ins_encode %{
16873     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
16874     if (tpc == NULL) {
16875       ciEnv::current()->record_failure("CodeCache is full");
16876       return;
16877     }
16878   %}
16879   ins_pipe( pipe_slow );
16880 %}
16881 
16882 // fast char[] to byte[] compression
16883 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16884                          vRegD_V0 tmp1, vRegD_V1 tmp2,
16885                          vRegD_V2 tmp3, vRegD_V3 tmp4,
16886                          iRegI_R0 result, rFlagsReg cr)
16887 %{
16888   match(Set result (StrCompressedCopy src (Binary dst len)));
16889   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16890 
16891   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
16892   ins_encode %{
16893     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16894                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16895                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
16896                            $result$$Register);
16897   %}
16898   ins_pipe( pipe_slow );
16899 %}
16900 
16901 // fast byte[] to char[] inflation
16902 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16903                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16904 %{
16905   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16906   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16907 
16908   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16909   ins_encode %{
16910     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16911                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16912                                         $tmp3$$FloatRegister, $tmp4$$Register);
16913     if (tpc == NULL) {
16914       ciEnv::current()->record_failure("CodeCache is full");
16915       return;
16916     }
16917   %}
16918   ins_pipe(pipe_class_memory);
16919 %}
16920 
16921 // encode char[] to byte[] in ISO_8859_1
16922 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16923                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
16924                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
16925                           iRegI_R0 result, rFlagsReg cr)
16926 %{
16927   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16928   match(Set result (EncodeISOArray src (Binary dst len)));
16929   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
16930          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
16931 
16932   format %{ "Encode array $src,$dst,$len -> $result" %}
16933   ins_encode %{
16934     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16935          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
16936          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
16937   %}
16938   ins_pipe( pipe_class_memory );
16939 %}
16940 
16941 // ============================================================================
16942 // This name is KNOWN by the ADLC and cannot be changed.
16943 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16944 // for this guy.
16945 instruct tlsLoadP(thread_RegP dst)
16946 %{
16947   match(Set dst (ThreadLocal));
16948 
16949   ins_cost(0);
16950 
16951   format %{ " -- \t// $dst=Thread::current(), empty" %}
16952 
16953   size(0);
16954 
16955   ins_encode( /*empty*/ );
16956 
16957   ins_pipe(pipe_class_empty);
16958 %}
16959 
16960 //----------PEEPHOLE RULES-----------------------------------------------------
16961 // These must follow all instruction definitions as they use the names
16962 // defined in the instructions definitions.
16963 //
16964 // peepmatch ( root_instr_name [preceding_instruction]* );
16965 //
16966 // peepconstraint %{
16967 // (instruction_number.operand_name relational_op instruction_number.operand_name
16968 //  [, ...] );
16969 // // instruction numbers are zero-based using left to right order in peepmatch
16970 //
16971 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
16972 // // provide an instruction_number.operand_name for each operand that appears
16973 // // in the replacement instruction's match rule
16974 //
16975 // ---------VM FLAGS---------------------------------------------------------
16976 //
16977 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16978 //
16979 // Each peephole rule is given an identifying number starting with zero and
16980 // increasing by one in the order seen by the parser.  An individual peephole
16981 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16982 // on the command-line.
16983 //
16984 // ---------CURRENT LIMITATIONS----------------------------------------------
16985 //
16986 // Only match adjacent instructions in same basic block
16987 // Only equality constraints
16988 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16989 // Only one replacement instruction
16990 //
16991 // ---------EXAMPLE----------------------------------------------------------
16992 //
16993 // // pertinent parts of existing instructions in architecture description
16994 // instruct movI(iRegINoSp dst, iRegI src)
16995 // %{
16996 //   match(Set dst (CopyI src));
16997 // %}
16998 //
16999 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17000 // %{
17001 //   match(Set dst (AddI dst src));
17002 //   effect(KILL cr);
17003 // %}
17004 //
17005 // // Change (inc mov) to lea
17006 // peephole %{
17007 //   // increment preceeded by register-register move
17008 //   peepmatch ( incI_iReg movI );
17009 //   // require that the destination register of the increment
17010 //   // match the destination register of the move
17011 //   peepconstraint ( 0.dst == 1.dst );
17012 //   // construct a replacement instruction that sets
17013 //   // the destination to ( move's source register + one )
17014 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17015 // %}
17016 //
17017 
17018 // Implementation no longer uses movX instructions since
17019 // machine-independent system no longer uses CopyX nodes.
17020 //
17021 // peephole
17022 // %{
17023 //   peepmatch (incI_iReg movI);
17024 //   peepconstraint (0.dst == 1.dst);
17025 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17026 // %}
17027 
17028 // peephole
17029 // %{
17030 //   peepmatch (decI_iReg movI);
17031 //   peepconstraint (0.dst == 1.dst);
17032 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17033 // %}
17034 
17035 // peephole
17036 // %{
17037 //   peepmatch (addI_iReg_imm movI);
17038 //   peepconstraint (0.dst == 1.dst);
17039 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17040 // %}
17041 
17042 // peephole
17043 // %{
17044 //   peepmatch (incL_iReg movL);
17045 //   peepconstraint (0.dst == 1.dst);
17046 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17047 // %}
17048 
17049 // peephole
17050 // %{
17051 //   peepmatch (decL_iReg movL);
17052 //   peepconstraint (0.dst == 1.dst);
17053 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17054 // %}
17055 
17056 // peephole
17057 // %{
17058 //   peepmatch (addL_iReg_imm movL);
17059 //   peepconstraint (0.dst == 1.dst);
17060 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17061 // %}
17062 
17063 // peephole
17064 // %{
17065 //   peepmatch (addP_iReg_imm movP);
17066 //   peepconstraint (0.dst == 1.dst);
17067 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17068 // %}
17069 
17070 // // Change load of spilled value to only a spill
17071 // instruct storeI(memory mem, iRegI src)
17072 // %{
17073 //   match(Set mem (StoreI mem src));
17074 // %}
17075 //
17076 // instruct loadI(iRegINoSp dst, memory mem)
17077 // %{
17078 //   match(Set dst (LoadI mem));
17079 // %}
17080 //
17081 
17082 //----------SMARTSPILL RULES---------------------------------------------------
17083 // These must follow all instruction definitions as they use the names
17084 // defined in the instructions definitions.
17085 
17086 // Local Variables:
17087 // mode: c++
17088 // End: