1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return 4 * NativeInstruction::instruction_size;
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298   bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // predicate controlling addressing modes
 1315   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1316 %}
 1317 
 1318 source %{
 1319 
 1320   // Derived RegMask with conditionally allocatable registers
 1321 
 1322   void PhaseOutput::pd_perform_mach_node_analysis() {
 1323   }
 1324 
 1325   int MachNode::pd_alignment_required() const {
 1326     return 1;
 1327   }
 1328 
 1329   int MachNode::compute_padding(int current_offset) const {
 1330     return 0;
 1331   }
 1332 
 1333   RegMask _ANY_REG32_mask;
 1334   RegMask _ANY_REG_mask;
 1335   RegMask _PTR_REG_mask;
 1336   RegMask _NO_SPECIAL_REG32_mask;
 1337   RegMask _NO_SPECIAL_REG_mask;
 1338   RegMask _NO_SPECIAL_PTR_REG_mask;
 1339 
 1340   void reg_mask_init() {
 1341     // We derive below RegMask(s) from the ones which are auto-generated from
 1342     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1343     // registers conditionally reserved.
 1344 
 1345     _ANY_REG32_mask = _ALL_REG32_mask;
 1346     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1347 
 1348     _ANY_REG_mask = _ALL_REG_mask;
 1349 
 1350     _PTR_REG_mask = _ALL_REG_mask;
 1351 
 1352     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1353     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1354 
 1355     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1356     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1357 
 1358     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     // r27 is not allocatable when compressed oops is on and heapbase is not
 1362     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1363     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1364       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1365       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1366       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1367     }
 1368 
 1369     // r29 is not allocatable when PreserveFramePointer is on
 1370     if (PreserveFramePointer) {
 1371       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1372       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1373       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1374     }
 1375   }
 1376 
 1377   // Optimizaton of volatile gets and puts
 1378   // -------------------------------------
 1379   //
 1380   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1381   // use to implement volatile reads and writes. For a volatile read
 1382   // we simply need
 1383   //
 1384   //   ldar<x>
 1385   //
 1386   // and for a volatile write we need
 1387   //
 1388   //   stlr<x>
 1389   //
 1390   // Alternatively, we can implement them by pairing a normal
 1391   // load/store with a memory barrier. For a volatile read we need
 1392   //
 1393   //   ldr<x>
 1394   //   dmb ishld
 1395   //
 1396   // for a volatile write
 1397   //
 1398   //   dmb ish
 1399   //   str<x>
 1400   //   dmb ish
 1401   //
 1402   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1403   // sequences. These are normally translated to an instruction
 1404   // sequence like the following
 1405   //
 1406   //   dmb      ish
 1407   // retry:
 1408   //   ldxr<x>   rval raddr
 1409   //   cmp       rval rold
 1410   //   b.ne done
 1411   //   stlxr<x>  rval, rnew, rold
 1412   //   cbnz      rval retry
 1413   // done:
 1414   //   cset      r0, eq
 1415   //   dmb ishld
 1416   //
 1417   // Note that the exclusive store is already using an stlxr
 1418   // instruction. That is required to ensure visibility to other
 1419   // threads of the exclusive write (assuming it succeeds) before that
 1420   // of any subsequent writes.
 1421   //
 1422   // The following instruction sequence is an improvement on the above
 1423   //
 1424   // retry:
 1425   //   ldaxr<x>  rval raddr
 1426   //   cmp       rval rold
 1427   //   b.ne done
 1428   //   stlxr<x>  rval, rnew, rold
 1429   //   cbnz      rval retry
 1430   // done:
 1431   //   cset      r0, eq
 1432   //
 1433   // We don't need the leading dmb ish since the stlxr guarantees
 1434   // visibility of prior writes in the case that the swap is
 1435   // successful. Crucially we don't have to worry about the case where
 1436   // the swap is not successful since no valid program should be
 1437   // relying on visibility of prior changes by the attempting thread
 1438   // in the case where the CAS fails.
 1439   //
 1440   // Similarly, we don't need the trailing dmb ishld if we substitute
 1441   // an ldaxr instruction since that will provide all the guarantees we
 1442   // require regarding observation of changes made by other threads
 1443   // before any change to the CAS address observed by the load.
 1444   //
 1445   // In order to generate the desired instruction sequence we need to
 1446   // be able to identify specific 'signature' ideal graph node
 1447   // sequences which i) occur as a translation of a volatile reads or
 1448   // writes or CAS operations and ii) do not occur through any other
 1449   // translation or graph transformation. We can then provide
 1450   // alternative aldc matching rules which translate these node
 1451   // sequences to the desired machine code sequences. Selection of the
 1452   // alternative rules can be implemented by predicates which identify
 1453   // the relevant node sequences.
 1454   //
 1455   // The ideal graph generator translates a volatile read to the node
 1456   // sequence
 1457   //
 1458   //   LoadX[mo_acquire]
 1459   //   MemBarAcquire
 1460   //
 1461   // As a special case when using the compressed oops optimization we
 1462   // may also see this variant
 1463   //
 1464   //   LoadN[mo_acquire]
 1465   //   DecodeN
 1466   //   MemBarAcquire
 1467   //
 1468   // A volatile write is translated to the node sequence
 1469   //
 1470   //   MemBarRelease
 1471   //   StoreX[mo_release] {CardMark}-optional
 1472   //   MemBarVolatile
 1473   //
 1474   // n.b. the above node patterns are generated with a strict
 1475   // 'signature' configuration of input and output dependencies (see
 1476   // the predicates below for exact details). The card mark may be as
 1477   // simple as a few extra nodes or, in a few GC configurations, may
 1478   // include more complex control flow between the leading and
 1479   // trailing memory barriers. However, whatever the card mark
 1480   // configuration these signatures are unique to translated volatile
 1481   // reads/stores -- they will not appear as a result of any other
 1482   // bytecode translation or inlining nor as a consequence of
 1483   // optimizing transforms.
 1484   //
 1485   // We also want to catch inlined unsafe volatile gets and puts and
 1486   // be able to implement them using either ldar<x>/stlr<x> or some
 1487   // combination of ldr<x>/stlr<x> and dmb instructions.
 1488   //
 1489   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1490   // normal volatile put node sequence containing an extra cpuorder
 1491   // membar
 1492   //
 1493   //   MemBarRelease
 1494   //   MemBarCPUOrder
 1495   //   StoreX[mo_release] {CardMark}-optional
 1496   //   MemBarCPUOrder
 1497   //   MemBarVolatile
 1498   //
 1499   // n.b. as an aside, a cpuorder membar is not itself subject to
 1500   // matching and translation by adlc rules.  However, the rule
 1501   // predicates need to detect its presence in order to correctly
 1502   // select the desired adlc rules.
 1503   //
 1504   // Inlined unsafe volatile gets manifest as a slightly different
 1505   // node sequence to a normal volatile get because of the
 1506   // introduction of some CPUOrder memory barriers to bracket the
 1507   // Load. However, but the same basic skeleton of a LoadX feeding a
 1508   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1509   // present
 1510   //
 1511   //   MemBarCPUOrder
 1512   //        ||       \\
 1513   //   MemBarCPUOrder LoadX[mo_acquire]
 1514   //        ||            |
 1515   //        ||       {DecodeN} optional
 1516   //        ||       /
 1517   //     MemBarAcquire
 1518   //
 1519   // In this case the acquire membar does not directly depend on the
 1520   // load. However, we can be sure that the load is generated from an
 1521   // inlined unsafe volatile get if we see it dependent on this unique
 1522   // sequence of membar nodes. Similarly, given an acquire membar we
 1523   // can know that it was added because of an inlined unsafe volatile
 1524   // get if it is fed and feeds a cpuorder membar and if its feed
 1525   // membar also feeds an acquiring load.
 1526   //
 1527   // Finally an inlined (Unsafe) CAS operation is translated to the
 1528   // following ideal graph
 1529   //
 1530   //   MemBarRelease
 1531   //   MemBarCPUOrder
 1532   //   CompareAndSwapX {CardMark}-optional
 1533   //   MemBarCPUOrder
 1534   //   MemBarAcquire
 1535   //
 1536   // So, where we can identify these volatile read and write
 1537   // signatures we can choose to plant either of the above two code
 1538   // sequences. For a volatile read we can simply plant a normal
 1539   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1540   // also choose to inhibit translation of the MemBarAcquire and
 1541   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1542   //
 1543   // When we recognise a volatile store signature we can choose to
 1544   // plant at a dmb ish as a translation for the MemBarRelease, a
 1545   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1546   // Alternatively, we can inhibit translation of the MemBarRelease
 1547   // and MemBarVolatile and instead plant a simple stlr<x>
 1548   // instruction.
 1549   //
 1550   // when we recognise a CAS signature we can choose to plant a dmb
 1551   // ish as a translation for the MemBarRelease, the conventional
 1552   // macro-instruction sequence for the CompareAndSwap node (which
 1553   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1554   // Alternatively, we can elide generation of the dmb instructions
 1555   // and plant the alternative CompareAndSwap macro-instruction
 1556   // sequence (which uses ldaxr<x>).
 1557   //
 1558   // Of course, the above only applies when we see these signature
 1559   // configurations. We still want to plant dmb instructions in any
 1560   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1561   // MemBarVolatile. For example, at the end of a constructor which
 1562   // writes final/volatile fields we will see a MemBarRelease
 1563   // instruction and this needs a 'dmb ish' lest we risk the
 1564   // constructed object being visible without making the
 1565   // final/volatile field writes visible.
 1566   //
 1567   // n.b. the translation rules below which rely on detection of the
 1568   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1569   // If we see anything other than the signature configurations we
 1570   // always just translate the loads and stores to ldr<x> and str<x>
 1571   // and translate acquire, release and volatile membars to the
 1572   // relevant dmb instructions.
 1573   //
 1574 
 1575   // is_CAS(int opcode, bool maybe_volatile)
 1576   //
 1577   // return true if opcode is one of the possible CompareAndSwapX
 1578   // values otherwise false.
 1579 
 1580   bool is_CAS(int opcode, bool maybe_volatile)
 1581   {
 1582     switch(opcode) {
 1583       // We handle these
 1584     case Op_CompareAndSwapI:
 1585     case Op_CompareAndSwapL:
 1586     case Op_CompareAndSwapP:
 1587     case Op_CompareAndSwapN:
 1588     case Op_ShenandoahCompareAndSwapP:
 1589     case Op_ShenandoahCompareAndSwapN:
 1590     case Op_CompareAndSwapB:
 1591     case Op_CompareAndSwapS:
 1592     case Op_GetAndSetI:
 1593     case Op_GetAndSetL:
 1594     case Op_GetAndSetP:
 1595     case Op_GetAndSetN:
 1596     case Op_GetAndAddI:
 1597     case Op_GetAndAddL:
 1598       return true;
 1599     case Op_CompareAndExchangeI:
 1600     case Op_CompareAndExchangeN:
 1601     case Op_CompareAndExchangeB:
 1602     case Op_CompareAndExchangeS:
 1603     case Op_CompareAndExchangeL:
 1604     case Op_CompareAndExchangeP:
 1605     case Op_WeakCompareAndSwapB:
 1606     case Op_WeakCompareAndSwapS:
 1607     case Op_WeakCompareAndSwapI:
 1608     case Op_WeakCompareAndSwapL:
 1609     case Op_WeakCompareAndSwapP:
 1610     case Op_WeakCompareAndSwapN:
 1611     case Op_ShenandoahWeakCompareAndSwapP:
 1612     case Op_ShenandoahWeakCompareAndSwapN:
 1613     case Op_ShenandoahCompareAndExchangeP:
 1614     case Op_ShenandoahCompareAndExchangeN:
 1615       return maybe_volatile;
 1616     default:
 1617       return false;
 1618     }
 1619   }
 1620 
 1621   // helper to determine the maximum number of Phi nodes we may need to
 1622   // traverse when searching from a card mark membar for the merge mem
 1623   // feeding a trailing membar or vice versa
 1624 
 1625 // predicates controlling emit of ldr<x>/ldar<x>
 1626 
 1627 bool unnecessary_acquire(const Node *barrier)
 1628 {
 1629   assert(barrier->is_MemBar(), "expecting a membar");
 1630 
 1631   MemBarNode* mb = barrier->as_MemBar();
 1632 
 1633   if (mb->trailing_load()) {
 1634     return true;
 1635   }
 1636 
 1637   if (mb->trailing_load_store()) {
 1638     Node* load_store = mb->in(MemBarNode::Precedent);
 1639     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1640     return is_CAS(load_store->Opcode(), true);
 1641   }
 1642 
 1643   return false;
 1644 }
 1645 
 1646 bool needs_acquiring_load(const Node *n)
 1647 {
 1648   assert(n->is_Load(), "expecting a load");
 1649   LoadNode *ld = n->as_Load();
 1650   return ld->is_acquire();
 1651 }
 1652 
 1653 bool unnecessary_release(const Node *n)
 1654 {
 1655   assert((n->is_MemBar() &&
 1656           n->Opcode() == Op_MemBarRelease),
 1657          "expecting a release membar");
 1658 
 1659   MemBarNode *barrier = n->as_MemBar();
 1660   if (!barrier->leading()) {
 1661     return false;
 1662   } else {
 1663     Node* trailing = barrier->trailing_membar();
 1664     MemBarNode* trailing_mb = trailing->as_MemBar();
 1665     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1666     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1667 
 1668     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1669     if (mem->is_Store()) {
 1670       assert(mem->as_Store()->is_release(), "");
 1671       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1672       return true;
 1673     } else {
 1674       assert(mem->is_LoadStore(), "");
 1675       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1676       return is_CAS(mem->Opcode(), true);
 1677     }
 1678   }
 1679   return false;
 1680 }
 1681 
 1682 bool unnecessary_volatile(const Node *n)
 1683 {
 1684   // assert n->is_MemBar();
 1685   MemBarNode *mbvol = n->as_MemBar();
 1686 
 1687   bool release = mbvol->trailing_store();
 1688   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1689 #ifdef ASSERT
 1690   if (release) {
 1691     Node* leading = mbvol->leading_membar();
 1692     assert(leading->Opcode() == Op_MemBarRelease, "");
 1693     assert(leading->as_MemBar()->leading_store(), "");
 1694     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1695   }
 1696 #endif
 1697 
 1698   return release;
 1699 }
 1700 
 1701 // predicates controlling emit of str<x>/stlr<x>
 1702 
 1703 bool needs_releasing_store(const Node *n)
 1704 {
 1705   // assert n->is_Store();
 1706   StoreNode *st = n->as_Store();
 1707   return st->trailing_membar() != NULL;
 1708 }
 1709 
 1710 // predicate controlling translation of CAS
 1711 //
 1712 // returns true if CAS needs to use an acquiring load otherwise false
 1713 
 1714 bool needs_acquiring_load_exclusive(const Node *n)
 1715 {
 1716   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1717   LoadStoreNode* ldst = n->as_LoadStore();
 1718   if (is_CAS(n->Opcode(), false)) {
 1719     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1720   } else {
 1721     return ldst->trailing_membar() != NULL;
 1722   }
 1723 
 1724   // so we can just return true here
 1725   return true;
 1726 }
 1727 
 1728 #define __ _masm.
 1729 
 1730 // advance declarations for helper functions to convert register
 1731 // indices to register objects
 1732 
 1733 // the ad file has to provide implementations of certain methods
 1734 // expected by the generic code
 1735 //
 1736 // REQUIRED FUNCTIONALITY
 1737 
 1738 //=============================================================================
 1739 
 1740 // !!!!! Special hack to get all types of calls to specify the byte offset
 1741 //       from the start of the call to the point where the return address
 1742 //       will point.
 1743 
 1744 int MachCallStaticJavaNode::ret_addr_offset()
 1745 {
 1746   // call should be a simple bl
 1747   int off = 4;
 1748   return off;
 1749 }
 1750 
 1751 int MachCallDynamicJavaNode::ret_addr_offset()
 1752 {
 1753   return 16; // movz, movk, movk, bl
 1754 }
 1755 
 1756 int MachCallRuntimeNode::ret_addr_offset() {
 1757   // for generated stubs the call will be
 1758   //   bl(addr)
 1759   // or with far branches
 1760   //   bl(trampoline_stub)
 1761   // for real runtime callouts it will be six instructions
 1762   // see aarch64_enc_java_to_runtime
 1763   //   adr(rscratch2, retaddr)
 1764   //   lea(rscratch1, RuntimeAddress(addr)
 1765   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1766   //   blr(rscratch1)
 1767   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1768   if (cb) {
 1769     return 1 * NativeInstruction::instruction_size;
 1770   } else {
 1771     return 6 * NativeInstruction::instruction_size;
 1772   }
 1773 }
 1774 
 1775 int MachCallNativeNode::ret_addr_offset() {
 1776   // This is implemented using aarch64_enc_java_to_runtime as above.
 1777   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1778   if (cb) {
 1779     return 1 * NativeInstruction::instruction_size;
 1780   } else {
 1781     return 6 * NativeInstruction::instruction_size;
 1782   }
 1783 }
 1784 
 1785 //=============================================================================
 1786 
 1787 #ifndef PRODUCT
 1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1789   st->print("BREAKPOINT");
 1790 }
 1791 #endif
 1792 
 1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1794   C2_MacroAssembler _masm(&cbuf);
 1795   __ brk(0);
 1796 }
 1797 
 1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1799   return MachNode::size(ra_);
 1800 }
 1801 
 1802 //=============================================================================
 1803 
 1804 #ifndef PRODUCT
 1805   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1806     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1807   }
 1808 #endif
 1809 
 1810   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1811     C2_MacroAssembler _masm(&cbuf);
 1812     for (int i = 0; i < _count; i++) {
 1813       __ nop();
 1814     }
 1815   }
 1816 
 1817   uint MachNopNode::size(PhaseRegAlloc*) const {
 1818     return _count * NativeInstruction::instruction_size;
 1819   }
 1820 
 1821 //=============================================================================
 1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1823 
 1824 int ConstantTable::calculate_table_base_offset() const {
 1825   return 0;  // absolute addressing, no offset
 1826 }
 1827 
 1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1830   ShouldNotReachHere();
 1831 }
 1832 
 1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1834   // Empty encoding
 1835 }
 1836 
 1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1838   return 0;
 1839 }
 1840 
 1841 #ifndef PRODUCT
 1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1843   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1844 }
 1845 #endif
 1846 
 1847 #ifndef PRODUCT
 1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1849   Compile* C = ra_->C;
 1850 
 1851   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1852 
 1853   if (C->output()->need_stack_bang(framesize))
 1854     st->print("# stack bang size=%d\n\t", framesize);
 1855 
 1856   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1857     st->print("sub  sp, sp, #%d\n\t", framesize);
 1858     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1859     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1860   } else {
 1861     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1862     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1863     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1864     st->print("sub  sp, sp, rscratch1");
 1865   }
 1866   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1867     st->print("\n\t");
 1868     st->print("ldr  rscratch1, [guard]\n\t");
 1869     st->print("dmb ishld\n\t");
 1870     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1871     st->print("cmp  rscratch1, rscratch2\n\t");
 1872     st->print("b.eq skip");
 1873     st->print("\n\t");
 1874     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1875     st->print("b skip\n\t");
 1876     st->print("guard: int\n\t");
 1877     st->print("\n\t");
 1878     st->print("skip:\n\t");
 1879   }
 1880 }
 1881 #endif
 1882 
 1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1884   Compile* C = ra_->C;
 1885   C2_MacroAssembler _masm(&cbuf);
 1886 
 1887   // n.b. frame size includes space for return pc and rfp
 1888   const int framesize = C->output()->frame_size_in_bytes();
 1889 
 1890   // insert a nop at the start of the prolog so we can patch in a
 1891   // branch if we need to invalidate the method later
 1892   __ nop();
 1893 
 1894   if (C->clinit_barrier_on_entry()) {
 1895     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1896 
 1897     Label L_skip_barrier;
 1898 
 1899     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1900     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1901     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1902     __ bind(L_skip_barrier);
 1903   }
 1904 
 1905   if (C->max_vector_size() > 0) {
 1906     __ reinitialize_ptrue();
 1907   }
 1908 
 1909   int bangsize = C->output()->bang_size_in_bytes();
 1910   if (C->output()->need_stack_bang(bangsize))
 1911     __ generate_stack_overflow_check(bangsize);
 1912 
 1913   __ build_frame(framesize);
 1914 
 1915   if (C->stub_function() == NULL) {
 1916     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1917     bs->nmethod_entry_barrier(&_masm);
 1918   }
 1919 
 1920   if (VerifyStackAtCalls) {
 1921     Unimplemented();
 1922   }
 1923 
 1924   C->output()->set_frame_complete(cbuf.insts_size());
 1925 
 1926   if (C->has_mach_constant_base_node()) {
 1927     // NOTE: We set the table base offset here because users might be
 1928     // emitted before MachConstantBaseNode.
 1929     ConstantTable& constant_table = C->output()->constant_table();
 1930     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1931   }
 1932 }
 1933 
 1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1935 {
 1936   return MachNode::size(ra_); // too many variables; just compute it
 1937                               // the hard way
 1938 }
 1939 
 1940 int MachPrologNode::reloc() const
 1941 {
 1942   return 0;
 1943 }
 1944 
 1945 //=============================================================================
 1946 
 1947 #ifndef PRODUCT
 1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1949   Compile* C = ra_->C;
 1950   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1951 
 1952   st->print("# pop frame %d\n\t",framesize);
 1953 
 1954   if (framesize == 0) {
 1955     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1956   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1957     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1958     st->print("add  sp, sp, #%d\n\t", framesize);
 1959   } else {
 1960     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, rscratch1\n\t");
 1962     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1963   }
 1964 
 1965   if (do_polling() && C->is_method_compilation()) {
 1966     st->print("# test polling word\n\t");
 1967     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1968     st->print("cmp  sp, rscratch1\n\t");
 1969     st->print("bhi #slow_path");
 1970   }
 1971 }
 1972 #endif
 1973 
 1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1975   Compile* C = ra_->C;
 1976   C2_MacroAssembler _masm(&cbuf);
 1977   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1978 
 1979   __ remove_frame(framesize);
 1980 
 1981   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1982     __ reserved_stack_check();
 1983   }
 1984 
 1985   if (do_polling() && C->is_method_compilation()) {
 1986     Label dummy_label;
 1987     Label* code_stub = &dummy_label;
 1988     if (!C->output()->in_scratch_emit_size()) {
 1989       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1990     }
 1991     __ relocate(relocInfo::poll_return_type);
 1992     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1993   }
 1994 }
 1995 
 1996 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1997   // Variable size. Determine dynamically.
 1998   return MachNode::size(ra_);
 1999 }
 2000 
 2001 int MachEpilogNode::reloc() const {
 2002   // Return number of relocatable values contained in this instruction.
 2003   return 1; // 1 for polling page.
 2004 }
 2005 
 2006 const Pipeline * MachEpilogNode::pipeline() const {
 2007   return MachNode::pipeline_class();
 2008 }
 2009 
 2010 //=============================================================================
 2011 
 2012 // Figure out which register class each belongs in: rc_int, rc_float or
 2013 // rc_stack.
 2014 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2015 
 2016 static enum RC rc_class(OptoReg::Name reg) {
 2017 
 2018   if (reg == OptoReg::Bad) {
 2019     return rc_bad;
 2020   }
 2021 
 2022   // we have 32 int registers * 2 halves
 2023   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2024 
 2025   if (reg < slots_of_int_registers) {
 2026     return rc_int;
 2027   }
 2028 
 2029   // we have 32 float register * 8 halves
 2030   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2031   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2032     return rc_float;
 2033   }
 2034 
 2035   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2036   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2037     return rc_predicate;
 2038   }
 2039 
 2040   // Between predicate regs & stack is the flags.
 2041   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2042 
 2043   return rc_stack;
 2044 }
 2045 
 2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2047   Compile* C = ra_->C;
 2048 
 2049   // Get registers to move.
 2050   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2051   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2052   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2053   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2054 
 2055   enum RC src_hi_rc = rc_class(src_hi);
 2056   enum RC src_lo_rc = rc_class(src_lo);
 2057   enum RC dst_hi_rc = rc_class(dst_hi);
 2058   enum RC dst_lo_rc = rc_class(dst_lo);
 2059 
 2060   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2061 
 2062   if (src_hi != OptoReg::Bad) {
 2063     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2064            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2065            "expected aligned-adjacent pairs");
 2066   }
 2067 
 2068   if (src_lo == dst_lo && src_hi == dst_hi) {
 2069     return 0;            // Self copy, no move.
 2070   }
 2071 
 2072   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2073               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2074   int src_offset = ra_->reg2offset(src_lo);
 2075   int dst_offset = ra_->reg2offset(dst_lo);
 2076 
 2077   if (bottom_type()->isa_vect() != NULL) {
 2078     uint ireg = ideal_reg();
 2079     if (ireg == Op_VecA && cbuf) {
 2080       C2_MacroAssembler _masm(cbuf);
 2081       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2082       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2083         // stack->stack
 2084         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2085                                                 sve_vector_reg_size_in_bytes);
 2086       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2087         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2088                             sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2090         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2091                               sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2093         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2094                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2095                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2096       } else {
 2097         ShouldNotReachHere();
 2098       }
 2099     } else if (cbuf) {
 2100       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2101       C2_MacroAssembler _masm(cbuf);
 2102       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2103       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2104         // stack->stack
 2105         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2106         if (ireg == Op_VecD) {
 2107           __ unspill(rscratch1, true, src_offset);
 2108           __ spill(rscratch1, true, dst_offset);
 2109         } else {
 2110           __ spill_copy128(src_offset, dst_offset);
 2111         }
 2112       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2113         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2114                ireg == Op_VecD ? __ T8B : __ T16B,
 2115                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2116       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2117         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2118                  ireg == Op_VecD ? __ D : __ Q,
 2119                  ra_->reg2offset(dst_lo));
 2120       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2121         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2122                    ireg == Op_VecD ? __ D : __ Q,
 2123                    ra_->reg2offset(src_lo));
 2124       } else {
 2125         ShouldNotReachHere();
 2126       }
 2127     }
 2128   } else if (cbuf) {
 2129     C2_MacroAssembler _masm(cbuf);
 2130     switch (src_lo_rc) {
 2131     case rc_int:
 2132       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2133         if (is64) {
 2134             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2135                    as_Register(Matcher::_regEncode[src_lo]));
 2136         } else {
 2137             C2_MacroAssembler _masm(cbuf);
 2138             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2139                     as_Register(Matcher::_regEncode[src_lo]));
 2140         }
 2141       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2142         if (is64) {
 2143             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2144                      as_Register(Matcher::_regEncode[src_lo]));
 2145         } else {
 2146             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2147                      as_Register(Matcher::_regEncode[src_lo]));
 2148         }
 2149       } else {                    // gpr --> stack spill
 2150         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2151         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2152       }
 2153       break;
 2154     case rc_float:
 2155       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2156         if (is64) {
 2157             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2158                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2159         } else {
 2160             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2161                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2162         }
 2163       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2164         if (is64) {
 2165             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2166                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2167         } else {
 2168             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         }
 2171       } else {                    // fpr --> stack spill
 2172         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2173         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2174                  is64 ? __ D : __ S, dst_offset);
 2175       }
 2176       break;
 2177     case rc_stack:
 2178       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2179         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2180       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2181         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2182                    is64 ? __ D : __ S, src_offset);
 2183       } else {                    // stack --> stack copy
 2184         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2185         __ unspill(rscratch1, is64, src_offset);
 2186         __ spill(rscratch1, is64, dst_offset);
 2187       }
 2188       break;
 2189     default:
 2190       assert(false, "bad rc_class for spill");
 2191       ShouldNotReachHere();
 2192     }
 2193   }
 2194 
 2195   if (st) {
 2196     st->print("spill ");
 2197     if (src_lo_rc == rc_stack) {
 2198       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2199     } else {
 2200       st->print("%s -> ", Matcher::regName[src_lo]);
 2201     }
 2202     if (dst_lo_rc == rc_stack) {
 2203       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2204     } else {
 2205       st->print("%s", Matcher::regName[dst_lo]);
 2206     }
 2207     if (bottom_type()->isa_vect() != NULL) {
 2208       int vsize = 0;
 2209       switch (ideal_reg()) {
 2210       case Op_VecD:
 2211         vsize = 64;
 2212         break;
 2213       case Op_VecX:
 2214         vsize = 128;
 2215         break;
 2216       case Op_VecA:
 2217         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2218         break;
 2219       default:
 2220         assert(false, "bad register type for spill");
 2221         ShouldNotReachHere();
 2222       }
 2223       st->print("\t# vector spill size = %d", vsize);
 2224     } else {
 2225       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2226     }
 2227   }
 2228 
 2229   return 0;
 2230 
 2231 }
 2232 
 2233 #ifndef PRODUCT
 2234 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2235   if (!ra_)
 2236     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2237   else
 2238     implementation(NULL, ra_, false, st);
 2239 }
 2240 #endif
 2241 
 2242 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2243   implementation(&cbuf, ra_, false, NULL);
 2244 }
 2245 
 2246 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2247   return MachNode::size(ra_);
 2248 }
 2249 
 2250 //=============================================================================
 2251 
 2252 #ifndef PRODUCT
 2253 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2254   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2255   int reg = ra_->get_reg_first(this);
 2256   st->print("add %s, rsp, #%d]\t# box lock",
 2257             Matcher::regName[reg], offset);
 2258 }
 2259 #endif
 2260 
 2261 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2262   C2_MacroAssembler _masm(&cbuf);
 2263 
 2264   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2265   int reg    = ra_->get_encode(this);
 2266 
 2267   // This add will handle any 24-bit signed offset. 24 bits allows an
 2268   // 8 megabyte stack frame.
 2269   __ add(as_Register(reg), sp, offset);
 2270 }
 2271 
 2272 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2273   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2274   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2275 
 2276   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2277     return NativeInstruction::instruction_size;
 2278   } else {
 2279     return 2 * NativeInstruction::instruction_size;
 2280   }
 2281 }
 2282 
 2283 //=============================================================================
 2284 
 2285 #ifndef PRODUCT
 2286 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2287 {
 2288   st->print_cr("# MachUEPNode");
 2289   if (UseCompressedClassPointers) {
 2290     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2291     if (CompressedKlassPointers::shift() != 0) {
 2292       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2293     }
 2294   } else {
 2295    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2296   }
 2297   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2298   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2299 }
 2300 #endif
 2301 
 2302 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2303 {
 2304   // This is the unverified entry point.
 2305   C2_MacroAssembler _masm(&cbuf);
 2306 
 2307   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2308   Label skip;
 2309   // TODO
 2310   // can we avoid this skip and still use a reloc?
 2311   __ br(Assembler::EQ, skip);
 2312   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2313   __ bind(skip);
 2314 }
 2315 
 2316 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2317 {
 2318   return MachNode::size(ra_);
 2319 }
 2320 
 2321 // REQUIRED EMIT CODE
 2322 
 2323 //=============================================================================
 2324 
 2325 // Emit exception handler code.
 2326 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2327 {
 2328   // mov rscratch1 #exception_blob_entry_point
 2329   // br rscratch1
 2330   // Note that the code buffer's insts_mark is always relative to insts.
 2331   // That's why we must use the macroassembler to generate a handler.
 2332   C2_MacroAssembler _masm(&cbuf);
 2333   address base = __ start_a_stub(size_exception_handler());
 2334   if (base == NULL) {
 2335     ciEnv::current()->record_failure("CodeCache is full");
 2336     return 0;  // CodeBuffer::expand failed
 2337   }
 2338   int offset = __ offset();
 2339   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2340   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2341   __ end_a_stub();
 2342   return offset;
 2343 }
 2344 
 2345 // Emit deopt handler code.
 2346 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2347 {
 2348   // Note that the code buffer's insts_mark is always relative to insts.
 2349   // That's why we must use the macroassembler to generate a handler.
 2350   C2_MacroAssembler _masm(&cbuf);
 2351   address base = __ start_a_stub(size_deopt_handler());
 2352   if (base == NULL) {
 2353     ciEnv::current()->record_failure("CodeCache is full");
 2354     return 0;  // CodeBuffer::expand failed
 2355   }
 2356   int offset = __ offset();
 2357 
 2358   __ adr(lr, __ pc());
 2359   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2360 
 2361   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
 2362   __ end_a_stub();
 2363   return offset;
 2364 }
 2365 
 2366 // REQUIRED MATCHER CODE
 2367 
 2368 //=============================================================================
 2369 
 2370 const bool Matcher::match_rule_supported(int opcode) {
 2371   if (!has_match_rule(opcode))
 2372     return false;
 2373 
 2374   bool ret_value = true;
 2375   switch (opcode) {
 2376     case Op_CacheWB:
 2377     case Op_CacheWBPreSync:
 2378     case Op_CacheWBPostSync:
 2379       if (!VM_Version::supports_data_cache_line_flush()) {
 2380         ret_value = false;
 2381       }
 2382       break;
 2383   }
 2384 
 2385   return ret_value; // Per default match rules are supported.
 2386 }
 2387 
 2388 // Identify extra cases that we might want to provide match rules for vector nodes and
 2389 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2390 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2391   if (!match_rule_supported(opcode)) {
 2392     return false;
 2393   }
 2394   int bit_size = vlen * type2aelembytes(bt) * 8;
 2395   if (UseSVE == 0 && bit_size > 128) {
 2396     return false;
 2397   }
 2398   if (UseSVE > 0) {
 2399     return op_sve_supported(opcode, vlen, bt);
 2400   } else { // NEON
 2401     // Special cases
 2402     switch (opcode) {
 2403     case Op_VectorMaskCmp:
 2404     // We don't have VectorReinterpret with bit_size less than 64 support for
 2405     // now, even for byte type. To be refined with fully VectorCast support.
 2406     case Op_VectorReinterpret:
 2407       if (vlen < 2 || bit_size < 64) {
 2408         return false;
 2409       }
 2410       break;
 2411     case Op_MulAddVS2VI:
 2412       if (bit_size < 128) {
 2413         return false;
 2414       }
 2415       break;
 2416     case Op_MulVL:
 2417       return false;
 2418     case Op_VectorLoadShuffle:
 2419     case Op_VectorRearrange:
 2420       if (vlen < 4) {
 2421         return false;
 2422       }
 2423       break;
 2424     // Some types of VectorCast are not implemented for now.
 2425     case Op_VectorCastI2X:
 2426       if (bt == T_BYTE) {
 2427         return false;
 2428       }
 2429       break;
 2430     case Op_VectorCastS2X:
 2431       if (vlen < 4 || bit_size < 64) {
 2432         return false;
 2433       }
 2434       break;
 2435     case Op_VectorCastF2X:
 2436     case Op_VectorCastD2X:
 2437       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2438         return false;
 2439       }
 2440       break;
 2441     case Op_LoadVectorGather:
 2442     case Op_StoreVectorScatter:
 2443       return false;
 2444     default:
 2445       break;
 2446     }
 2447   }
 2448   return vector_size_supported(bt, vlen);
 2449 }
 2450 
 2451 const RegMask* Matcher::predicate_reg_mask(void) {
 2452   return &_PR_REG_mask;
 2453 }
 2454 
 2455 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2456   return new TypeVectMask(elemTy, length);
 2457 }
 2458 
 2459 // Vector calling convention not yet implemented.
 2460 const bool Matcher::supports_vector_calling_convention(void) {
 2461   return false;
 2462 }
 2463 
 2464 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2465   Unimplemented();
 2466   return OptoRegPair(0, 0);
 2467 }
 2468 
 2469 // Is this branch offset short enough that a short branch can be used?
 2470 //
 2471 // NOTE: If the platform does not provide any short branch variants, then
 2472 //       this method should return false for offset 0.
 2473 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2474   // The passed offset is relative to address of the branch.
 2475 
 2476   return (-32768 <= offset && offset < 32768);
 2477 }
 2478 
 2479 // Vector width in bytes.
 2480 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2481   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2482   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2483   // Minimum 2 values in vector
 2484   if (size < 2*type2aelembytes(bt)) size = 0;
 2485   // But never < 4
 2486   if (size < 4) size = 0;
 2487   return size;
 2488 }
 2489 
 2490 // Limits on vector size (number of elements) loaded into vector.
 2491 const int Matcher::max_vector_size(const BasicType bt) {
 2492   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2493 }
 2494 
 2495 const int Matcher::min_vector_size(const BasicType bt) {
 2496   int max_size = max_vector_size(bt);
 2497   // Limit the min vector size to 8 bytes.
 2498   int size = 8 / type2aelembytes(bt);
 2499   if (bt == T_BYTE) {
 2500     // To support vector api shuffle/rearrange.
 2501     size = 4;
 2502   } else if (bt == T_BOOLEAN) {
 2503     // To support vector api load/store mask.
 2504     size = 2;
 2505   }
 2506   if (size < 2) size = 2;
 2507   return MIN2(size, max_size);
 2508 }
 2509 
 2510 // Actual max scalable vector register length.
 2511 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2512   return Matcher::max_vector_size(bt);
 2513 }
 2514 
 2515 // Vector ideal reg.
 2516 const uint Matcher::vector_ideal_reg(int len) {
 2517   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2518     return Op_VecA;
 2519   }
 2520   switch(len) {
 2521     // For 16-bit/32-bit mask vector, reuse VecD.
 2522     case  2:
 2523     case  4:
 2524     case  8: return Op_VecD;
 2525     case 16: return Op_VecX;
 2526   }
 2527   ShouldNotReachHere();
 2528   return 0;
 2529 }
 2530 
 2531 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2532   ShouldNotReachHere(); // generic vector operands not supported
 2533   return NULL;
 2534 }
 2535 
 2536 bool Matcher::is_reg2reg_move(MachNode* m) {
 2537   ShouldNotReachHere();  // generic vector operands not supported
 2538   return false;
 2539 }
 2540 
 2541 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2542   ShouldNotReachHere();  // generic vector operands not supported
 2543   return false;
 2544 }
 2545 
 2546 // Return whether or not this register is ever used as an argument.
 2547 // This function is used on startup to build the trampoline stubs in
 2548 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2549 // call in the trampoline, and arguments in those registers not be
 2550 // available to the callee.
 2551 bool Matcher::can_be_java_arg(int reg)
 2552 {
 2553   return
 2554     reg ==  R0_num || reg == R0_H_num ||
 2555     reg ==  R1_num || reg == R1_H_num ||
 2556     reg ==  R2_num || reg == R2_H_num ||
 2557     reg ==  R3_num || reg == R3_H_num ||
 2558     reg ==  R4_num || reg == R4_H_num ||
 2559     reg ==  R5_num || reg == R5_H_num ||
 2560     reg ==  R6_num || reg == R6_H_num ||
 2561     reg ==  R7_num || reg == R7_H_num ||
 2562     reg ==  V0_num || reg == V0_H_num ||
 2563     reg ==  V1_num || reg == V1_H_num ||
 2564     reg ==  V2_num || reg == V2_H_num ||
 2565     reg ==  V3_num || reg == V3_H_num ||
 2566     reg ==  V4_num || reg == V4_H_num ||
 2567     reg ==  V5_num || reg == V5_H_num ||
 2568     reg ==  V6_num || reg == V6_H_num ||
 2569     reg ==  V7_num || reg == V7_H_num;
 2570 }
 2571 
 2572 bool Matcher::is_spillable_arg(int reg)
 2573 {
 2574   return can_be_java_arg(reg);
 2575 }
 2576 
 2577 uint Matcher::int_pressure_limit()
 2578 {
 2579   // JDK-8183543: When taking the number of available registers as int
 2580   // register pressure threshold, the jtreg test:
 2581   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2582   // failed due to C2 compilation failure with
 2583   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2584   //
 2585   // A derived pointer is live at CallNode and then is flagged by RA
 2586   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2587   // derived pointers and lastly fail to spill after reaching maximum
 2588   // number of iterations. Lowering the default pressure threshold to
 2589   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2590   // a high register pressure area of the code so that split_DEF can
 2591   // generate DefinitionSpillCopy for the derived pointer.
 2592   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2593   if (!PreserveFramePointer) {
 2594     // When PreserveFramePointer is off, frame pointer is allocatable,
 2595     // but different from other SOC registers, it is excluded from
 2596     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2597     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2598     // See check_pressure_at_fatproj().
 2599     default_int_pressure_threshold--;
 2600   }
 2601   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2602 }
 2603 
 2604 uint Matcher::float_pressure_limit()
 2605 {
 2606   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2607   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2608 }
 2609 
 2610 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2611   return false;
 2612 }
 2613 
 2614 RegMask Matcher::divI_proj_mask() {
 2615   ShouldNotReachHere();
 2616   return RegMask();
 2617 }
 2618 
 2619 // Register for MODI projection of divmodI.
 2620 RegMask Matcher::modI_proj_mask() {
 2621   ShouldNotReachHere();
 2622   return RegMask();
 2623 }
 2624 
 2625 // Register for DIVL projection of divmodL.
 2626 RegMask Matcher::divL_proj_mask() {
 2627   ShouldNotReachHere();
 2628   return RegMask();
 2629 }
 2630 
 2631 // Register for MODL projection of divmodL.
 2632 RegMask Matcher::modL_proj_mask() {
 2633   ShouldNotReachHere();
 2634   return RegMask();
 2635 }
 2636 
 2637 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2638   return FP_REG_mask();
 2639 }
 2640 
 2641 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2642   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2643     Node* u = addp->fast_out(i);
 2644     if (u->is_LoadStore()) {
 2645       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2646       // instructions) only take register indirect as an operand, so
 2647       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2648       // must fail.
 2649       return false;
 2650     }
 2651     if (u->is_Mem()) {
 2652       int opsize = u->as_Mem()->memory_size();
 2653       assert(opsize > 0, "unexpected memory operand size");
 2654       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2655         return false;
 2656       }
 2657     }
 2658   }
 2659   return true;
 2660 }
 2661 
 2662 // Should the matcher clone input 'm' of node 'n'?
 2663 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2664   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
 2665     mstack.push(m, Visit);           // m = ShiftCntV
 2666     return true;
 2667   }
 2668   return false;
 2669 }
 2670 
 2671 // Should the Matcher clone shifts on addressing modes, expecting them
 2672 // to be subsumed into complex addressing expressions or compute them
 2673 // into registers?
 2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2675   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2676     return true;
 2677   }
 2678 
 2679   Node *off = m->in(AddPNode::Offset);
 2680   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2681       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2682       // Are there other uses besides address expressions?
 2683       !is_visited(off)) {
 2684     address_visited.set(off->_idx); // Flag as address_visited
 2685     mstack.push(off->in(2), Visit);
 2686     Node *conv = off->in(1);
 2687     if (conv->Opcode() == Op_ConvI2L &&
 2688         // Are there other uses besides address expressions?
 2689         !is_visited(conv)) {
 2690       address_visited.set(conv->_idx); // Flag as address_visited
 2691       mstack.push(conv->in(1), Pre_Visit);
 2692     } else {
 2693       mstack.push(conv, Pre_Visit);
 2694     }
 2695     address_visited.test_set(m->_idx); // Flag as address_visited
 2696     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2697     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2698     return true;
 2699   } else if (off->Opcode() == Op_ConvI2L &&
 2700              // Are there other uses besides address expressions?
 2701              !is_visited(off)) {
 2702     address_visited.test_set(m->_idx); // Flag as address_visited
 2703     address_visited.set(off->_idx); // Flag as address_visited
 2704     mstack.push(off->in(1), Pre_Visit);
 2705     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2706     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2707     return true;
 2708   }
 2709   return false;
 2710 }
 2711 
 2712 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2713   C2_MacroAssembler _masm(&cbuf);                                       \
 2714   {                                                                     \
 2715     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2716     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2717     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2718     __ INSN(REG, as_Register(BASE));                                    \
 2719   }
 2720 
 2721 
 2722 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2723   {
 2724     Address::extend scale;
 2725 
 2726     // Hooboy, this is fugly.  We need a way to communicate to the
 2727     // encoder that the index needs to be sign extended, so we have to
 2728     // enumerate all the cases.
 2729     switch (opcode) {
 2730     case INDINDEXSCALEDI2L:
 2731     case INDINDEXSCALEDI2LN:
 2732     case INDINDEXI2L:
 2733     case INDINDEXI2LN:
 2734       scale = Address::sxtw(size);
 2735       break;
 2736     default:
 2737       scale = Address::lsl(size);
 2738     }
 2739 
 2740     if (index == -1) {
 2741       return Address(base, disp);
 2742     } else {
 2743       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2744       return Address(base, as_Register(index), scale);
 2745     }
 2746   }
 2747 
 2748 
 2749 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2750 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2751 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2752 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2753                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2754 
 2755   // Used for all non-volatile memory accesses.  The use of
 2756   // $mem->opcode() to discover whether this pattern uses sign-extended
 2757   // offsets is something of a kludge.
 2758   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2759                         Register reg, int opcode,
 2760                         Register base, int index, int scale, int disp,
 2761                         int size_in_memory)
 2762   {
 2763     Address addr = mem2address(opcode, base, index, scale, disp);
 2764     if (addr.getMode() == Address::base_plus_offset) {
 2765       /* If we get an out-of-range offset it is a bug in the compiler,
 2766          so we assert here. */
 2767       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2768              "c2 compiler bug");
 2769       /* Fix up any out-of-range offsets. */
 2770       assert_different_registers(rscratch1, base);
 2771       assert_different_registers(rscratch1, reg);
 2772       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2773     }
 2774     (masm.*insn)(reg, addr);
 2775   }
 2776 
 2777   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2778                         FloatRegister reg, int opcode,
 2779                         Register base, int index, int size, int disp,
 2780                         int size_in_memory)
 2781   {
 2782     Address::extend scale;
 2783 
 2784     switch (opcode) {
 2785     case INDINDEXSCALEDI2L:
 2786     case INDINDEXSCALEDI2LN:
 2787       scale = Address::sxtw(size);
 2788       break;
 2789     default:
 2790       scale = Address::lsl(size);
 2791     }
 2792 
 2793     if (index == -1) {
 2794       /* If we get an out-of-range offset it is a bug in the compiler,
 2795          so we assert here. */
 2796       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2797       /* Fix up any out-of-range offsets. */
 2798       assert_different_registers(rscratch1, base);
 2799       Address addr = Address(base, disp);
 2800       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2801       (masm.*insn)(reg, addr);
 2802     } else {
 2803       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2804       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2805     }
 2806   }
 2807 
 2808   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2809                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2810                         int opcode, Register base, int index, int size, int disp)
 2811   {
 2812     if (index == -1) {
 2813       (masm.*insn)(reg, T, Address(base, disp));
 2814     } else {
 2815       assert(disp == 0, "unsupported address mode");
 2816       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2817     }
 2818   }
 2819 
 2820 %}
 2821 
 2822 
 2823 
 2824 //----------ENCODING BLOCK-----------------------------------------------------
 2825 // This block specifies the encoding classes used by the compiler to
 2826 // output byte streams.  Encoding classes are parameterized macros
 2827 // used by Machine Instruction Nodes in order to generate the bit
 2828 // encoding of the instruction.  Operands specify their base encoding
 2829 // interface with the interface keyword.  There are currently
 2830 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2831 // COND_INTER.  REG_INTER causes an operand to generate a function
 2832 // which returns its register number when queried.  CONST_INTER causes
 2833 // an operand to generate a function which returns the value of the
 2834 // constant when queried.  MEMORY_INTER causes an operand to generate
 2835 // four functions which return the Base Register, the Index Register,
 2836 // the Scale Value, and the Offset Value of the operand when queried.
 2837 // COND_INTER causes an operand to generate six functions which return
 2838 // the encoding code (ie - encoding bits for the instruction)
 2839 // associated with each basic boolean condition for a conditional
 2840 // instruction.
 2841 //
 2842 // Instructions specify two basic values for encoding.  Again, a
 2843 // function is available to check if the constant displacement is an
 2844 // oop. They use the ins_encode keyword to specify their encoding
 2845 // classes (which must be a sequence of enc_class names, and their
 2846 // parameters, specified in the encoding block), and they use the
 2847 // opcode keyword to specify, in order, their primary, secondary, and
 2848 // tertiary opcode.  Only the opcode sections which a particular
 2849 // instruction needs for encoding need to be specified.
 2850 encode %{
 2851   // Build emit functions for each basic byte or larger field in the
 2852   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2853   // from C++ code in the enc_class source block.  Emit functions will
 2854   // live in the main source block for now.  In future, we can
 2855   // generalize this by adding a syntax that specifies the sizes of
 2856   // fields in an order, so that the adlc can build the emit functions
 2857   // automagically
 2858 
 2859   // catch all for unimplemented encodings
 2860   enc_class enc_unimplemented %{
 2861     C2_MacroAssembler _masm(&cbuf);
 2862     __ unimplemented("C2 catch all");
 2863   %}
 2864 
 2865   // BEGIN Non-volatile memory access
 2866 
 2867   // This encoding class is generated automatically from ad_encode.m4.
 2868   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2869   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2870     Register dst_reg = as_Register($dst$$reg);
 2871     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2872                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2873   %}
 2874 
 2875   // This encoding class is generated automatically from ad_encode.m4.
 2876   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2877   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2878     Register dst_reg = as_Register($dst$$reg);
 2879     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2880                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2881   %}
 2882 
 2883   // This encoding class is generated automatically from ad_encode.m4.
 2884   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2885   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2886     Register dst_reg = as_Register($dst$$reg);
 2887     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2888                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2889   %}
 2890 
 2891   // This encoding class is generated automatically from ad_encode.m4.
 2892   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2893   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2894     Register dst_reg = as_Register($dst$$reg);
 2895     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2896                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2897   %}
 2898 
 2899   // This encoding class is generated automatically from ad_encode.m4.
 2900   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2901   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2902     Register dst_reg = as_Register($dst$$reg);
 2903     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2904                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2905   %}
 2906 
 2907   // This encoding class is generated automatically from ad_encode.m4.
 2908   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2909   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2910     Register dst_reg = as_Register($dst$$reg);
 2911     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2912                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2913   %}
 2914 
 2915   // This encoding class is generated automatically from ad_encode.m4.
 2916   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2917   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2918     Register dst_reg = as_Register($dst$$reg);
 2919     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2921   %}
 2922 
 2923   // This encoding class is generated automatically from ad_encode.m4.
 2924   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2925   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2926     Register dst_reg = as_Register($dst$$reg);
 2927     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2928                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2929   %}
 2930 
 2931   // This encoding class is generated automatically from ad_encode.m4.
 2932   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2933   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2934     Register dst_reg = as_Register($dst$$reg);
 2935     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2936                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2937   %}
 2938 
 2939   // This encoding class is generated automatically from ad_encode.m4.
 2940   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2941   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2942     Register dst_reg = as_Register($dst$$reg);
 2943     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2945   %}
 2946 
 2947   // This encoding class is generated automatically from ad_encode.m4.
 2948   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2949   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2950     Register dst_reg = as_Register($dst$$reg);
 2951     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2952                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2953   %}
 2954 
 2955   // This encoding class is generated automatically from ad_encode.m4.
 2956   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2957   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2958     Register dst_reg = as_Register($dst$$reg);
 2959     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2960                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2961   %}
 2962 
 2963   // This encoding class is generated automatically from ad_encode.m4.
 2964   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2965   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2966     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2967     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2968                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2969   %}
 2970 
 2971   // This encoding class is generated automatically from ad_encode.m4.
 2972   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2973   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2974     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2975     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2976                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2977   %}
 2978 
 2979   // This encoding class is generated automatically from ad_encode.m4.
 2980   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2981   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2982     Register src_reg = as_Register($src$$reg);
 2983     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2984                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2985   %}
 2986 
 2987   // This encoding class is generated automatically from ad_encode.m4.
 2988   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2989   enc_class aarch64_enc_strb0(memory1 mem) %{
 2990     C2_MacroAssembler _masm(&cbuf);
 2991     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2992                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2993   %}
 2994 
 2995   // This encoding class is generated automatically from ad_encode.m4.
 2996   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2997   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2998     Register src_reg = as_Register($src$$reg);
 2999     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3000                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3001   %}
 3002 
 3003   // This encoding class is generated automatically from ad_encode.m4.
 3004   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3005   enc_class aarch64_enc_strh0(memory2 mem) %{
 3006     C2_MacroAssembler _masm(&cbuf);
 3007     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3008                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3009   %}
 3010 
 3011   // This encoding class is generated automatically from ad_encode.m4.
 3012   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3013   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3014     Register src_reg = as_Register($src$$reg);
 3015     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3016                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3017   %}
 3018 
 3019   // This encoding class is generated automatically from ad_encode.m4.
 3020   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3021   enc_class aarch64_enc_strw0(memory4 mem) %{
 3022     C2_MacroAssembler _masm(&cbuf);
 3023     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3024                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3025   %}
 3026 
 3027   // This encoding class is generated automatically from ad_encode.m4.
 3028   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3029   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3030     Register src_reg = as_Register($src$$reg);
 3031     // we sometimes get asked to store the stack pointer into the
 3032     // current thread -- we cannot do that directly on AArch64
 3033     if (src_reg == r31_sp) {
 3034       C2_MacroAssembler _masm(&cbuf);
 3035       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3036       __ mov(rscratch2, sp);
 3037       src_reg = rscratch2;
 3038     }
 3039     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3041   %}
 3042 
 3043   // This encoding class is generated automatically from ad_encode.m4.
 3044   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3045   enc_class aarch64_enc_str0(memory8 mem) %{
 3046     C2_MacroAssembler _masm(&cbuf);
 3047     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3048                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3049   %}
 3050 
 3051   // This encoding class is generated automatically from ad_encode.m4.
 3052   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3053   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3054     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3055     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3056                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3057   %}
 3058 
 3059   // This encoding class is generated automatically from ad_encode.m4.
 3060   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3061   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3062     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3063     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3064                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3065   %}
 3066 
 3067   // This encoding class is generated automatically from ad_encode.m4.
 3068   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3069   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3070       C2_MacroAssembler _masm(&cbuf);
 3071       __ membar(Assembler::StoreStore);
 3072       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3073                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3074   %}
 3075 
 3076   // END Non-volatile memory access
 3077 
 3078   // Vector loads and stores
 3079   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3080     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3081     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3082        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3083   %}
 3084 
 3085   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3086     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3087     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3088        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3089   %}
 3090 
 3091   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3092     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3093     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3094        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3095   %}
 3096 
 3097   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3098     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3099     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3100        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3101   %}
 3102 
 3103   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3104     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3105     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3106        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3107   %}
 3108 
 3109   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3110     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3111     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3112        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3113   %}
 3114 
 3115   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3116     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3117     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3118        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3122     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3123     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3124        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3125   %}
 3126 
 3127   // volatile loads and stores
 3128 
 3129   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3130     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3131                  rscratch1, stlrb);
 3132   %}
 3133 
 3134   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3135     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136                  rscratch1, stlrh);
 3137   %}
 3138 
 3139   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3140     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3141                  rscratch1, stlrw);
 3142   %}
 3143 
 3144 
 3145   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3146     Register dst_reg = as_Register($dst$$reg);
 3147     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3148              rscratch1, ldarb);
 3149     __ sxtbw(dst_reg, dst_reg);
 3150   %}
 3151 
 3152   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3153     Register dst_reg = as_Register($dst$$reg);
 3154     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3155              rscratch1, ldarb);
 3156     __ sxtb(dst_reg, dst_reg);
 3157   %}
 3158 
 3159   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3160     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3161              rscratch1, ldarb);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3165     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3166              rscratch1, ldarb);
 3167   %}
 3168 
 3169   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3170     Register dst_reg = as_Register($dst$$reg);
 3171     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3172              rscratch1, ldarh);
 3173     __ sxthw(dst_reg, dst_reg);
 3174   %}
 3175 
 3176   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3177     Register dst_reg = as_Register($dst$$reg);
 3178     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3179              rscratch1, ldarh);
 3180     __ sxth(dst_reg, dst_reg);
 3181   %}
 3182 
 3183   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3184     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3185              rscratch1, ldarh);
 3186   %}
 3187 
 3188   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3189     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3190              rscratch1, ldarh);
 3191   %}
 3192 
 3193   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3194     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3195              rscratch1, ldarw);
 3196   %}
 3197 
 3198   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3199     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3200              rscratch1, ldarw);
 3201   %}
 3202 
 3203   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3204     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3205              rscratch1, ldar);
 3206   %}
 3207 
 3208   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3209     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3210              rscratch1, ldarw);
 3211     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3212   %}
 3213 
 3214   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3215     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3216              rscratch1, ldar);
 3217     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3218   %}
 3219 
 3220   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3221     Register src_reg = as_Register($src$$reg);
 3222     // we sometimes get asked to store the stack pointer into the
 3223     // current thread -- we cannot do that directly on AArch64
 3224     if (src_reg == r31_sp) {
 3225       C2_MacroAssembler _masm(&cbuf);
 3226       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3227       __ mov(rscratch2, sp);
 3228       src_reg = rscratch2;
 3229     }
 3230     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3231                  rscratch1, stlr);
 3232   %}
 3233 
 3234   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3235     {
 3236       C2_MacroAssembler _masm(&cbuf);
 3237       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3238       __ fmovs(rscratch2, src_reg);
 3239     }
 3240     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3241                  rscratch1, stlrw);
 3242   %}
 3243 
 3244   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3245     {
 3246       C2_MacroAssembler _masm(&cbuf);
 3247       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3248       __ fmovd(rscratch2, src_reg);
 3249     }
 3250     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3251                  rscratch1, stlr);
 3252   %}
 3253 
 3254   // synchronized read/update encodings
 3255 
 3256   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3257     C2_MacroAssembler _masm(&cbuf);
 3258     Register dst_reg = as_Register($dst$$reg);
 3259     Register base = as_Register($mem$$base);
 3260     int index = $mem$$index;
 3261     int scale = $mem$$scale;
 3262     int disp = $mem$$disp;
 3263     if (index == -1) {
 3264        if (disp != 0) {
 3265         __ lea(rscratch1, Address(base, disp));
 3266         __ ldaxr(dst_reg, rscratch1);
 3267       } else {
 3268         // TODO
 3269         // should we ever get anything other than this case?
 3270         __ ldaxr(dst_reg, base);
 3271       }
 3272     } else {
 3273       Register index_reg = as_Register(index);
 3274       if (disp == 0) {
 3275         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3276         __ ldaxr(dst_reg, rscratch1);
 3277       } else {
 3278         __ lea(rscratch1, Address(base, disp));
 3279         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3280         __ ldaxr(dst_reg, rscratch1);
 3281       }
 3282     }
 3283   %}
 3284 
 3285   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3286     C2_MacroAssembler _masm(&cbuf);
 3287     Register src_reg = as_Register($src$$reg);
 3288     Register base = as_Register($mem$$base);
 3289     int index = $mem$$index;
 3290     int scale = $mem$$scale;
 3291     int disp = $mem$$disp;
 3292     if (index == -1) {
 3293        if (disp != 0) {
 3294         __ lea(rscratch2, Address(base, disp));
 3295         __ stlxr(rscratch1, src_reg, rscratch2);
 3296       } else {
 3297         // TODO
 3298         // should we ever get anything other than this case?
 3299         __ stlxr(rscratch1, src_reg, base);
 3300       }
 3301     } else {
 3302       Register index_reg = as_Register(index);
 3303       if (disp == 0) {
 3304         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3305         __ stlxr(rscratch1, src_reg, rscratch2);
 3306       } else {
 3307         __ lea(rscratch2, Address(base, disp));
 3308         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3309         __ stlxr(rscratch1, src_reg, rscratch2);
 3310       }
 3311     }
 3312     __ cmpw(rscratch1, zr);
 3313   %}
 3314 
 3315   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3316     C2_MacroAssembler _masm(&cbuf);
 3317     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3318     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3319                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3320                /*weak*/ false, noreg);
 3321   %}
 3322 
 3323   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3324     C2_MacroAssembler _masm(&cbuf);
 3325     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3326     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3327                Assembler::word, /*acquire*/ false, /*release*/ true,
 3328                /*weak*/ false, noreg);
 3329   %}
 3330 
 3331   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3332     C2_MacroAssembler _masm(&cbuf);
 3333     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3334     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3335                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3336                /*weak*/ false, noreg);
 3337   %}
 3338 
 3339   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3340     C2_MacroAssembler _masm(&cbuf);
 3341     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3342     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3343                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3344                /*weak*/ false, noreg);
 3345   %}
 3346 
 3347 
 3348   // The only difference between aarch64_enc_cmpxchg and
 3349   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3350   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3351   // lock.
 3352   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3353     C2_MacroAssembler _masm(&cbuf);
 3354     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3355     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3356                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3357                /*weak*/ false, noreg);
 3358   %}
 3359 
 3360   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3361     C2_MacroAssembler _masm(&cbuf);
 3362     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3363     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3364                Assembler::word, /*acquire*/ true, /*release*/ true,
 3365                /*weak*/ false, noreg);
 3366   %}
 3367 
 3368   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3369     C2_MacroAssembler _masm(&cbuf);
 3370     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3371     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3372                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3373                /*weak*/ false, noreg);
 3374   %}
 3375 
 3376   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3377     C2_MacroAssembler _masm(&cbuf);
 3378     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3379     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3380                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3381                /*weak*/ false, noreg);
 3382   %}
 3383 
 3384   // auxiliary used for CompareAndSwapX to set result register
 3385   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3386     C2_MacroAssembler _masm(&cbuf);
 3387     Register res_reg = as_Register($res$$reg);
 3388     __ cset(res_reg, Assembler::EQ);
 3389   %}
 3390 
 3391   // prefetch encodings
 3392 
 3393   enc_class aarch64_enc_prefetchw(memory mem) %{
 3394     C2_MacroAssembler _masm(&cbuf);
 3395     Register base = as_Register($mem$$base);
 3396     int index = $mem$$index;
 3397     int scale = $mem$$scale;
 3398     int disp = $mem$$disp;
 3399     if (index == -1) {
 3400       __ prfm(Address(base, disp), PSTL1KEEP);
 3401     } else {
 3402       Register index_reg = as_Register(index);
 3403       if (disp == 0) {
 3404         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3405       } else {
 3406         __ lea(rscratch1, Address(base, disp));
 3407 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3408       }
 3409     }
 3410   %}
 3411 
 3412   /// mov envcodings
 3413 
 3414   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3415     C2_MacroAssembler _masm(&cbuf);
 3416     uint32_t con = (uint32_t)$src$$constant;
 3417     Register dst_reg = as_Register($dst$$reg);
 3418     if (con == 0) {
 3419       __ movw(dst_reg, zr);
 3420     } else {
 3421       __ movw(dst_reg, con);
 3422     }
 3423   %}
 3424 
 3425   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3426     C2_MacroAssembler _masm(&cbuf);
 3427     Register dst_reg = as_Register($dst$$reg);
 3428     uint64_t con = (uint64_t)$src$$constant;
 3429     if (con == 0) {
 3430       __ mov(dst_reg, zr);
 3431     } else {
 3432       __ mov(dst_reg, con);
 3433     }
 3434   %}
 3435 
 3436   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3437     C2_MacroAssembler _masm(&cbuf);
 3438     Register dst_reg = as_Register($dst$$reg);
 3439     address con = (address)$src$$constant;
 3440     if (con == NULL || con == (address)1) {
 3441       ShouldNotReachHere();
 3442     } else {
 3443       relocInfo::relocType rtype = $src->constant_reloc();
 3444       if (rtype == relocInfo::oop_type) {
 3445         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3446       } else if (rtype == relocInfo::metadata_type) {
 3447         __ mov_metadata(dst_reg, (Metadata*)con);
 3448       } else {
 3449         assert(rtype == relocInfo::none, "unexpected reloc type");
 3450         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3451           __ mov(dst_reg, con);
 3452         } else {
 3453           uint64_t offset;
 3454           __ adrp(dst_reg, con, offset);
 3455           __ add(dst_reg, dst_reg, offset);
 3456         }
 3457       }
 3458     }
 3459   %}
 3460 
 3461   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3462     C2_MacroAssembler _masm(&cbuf);
 3463     Register dst_reg = as_Register($dst$$reg);
 3464     __ mov(dst_reg, zr);
 3465   %}
 3466 
 3467   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3468     C2_MacroAssembler _masm(&cbuf);
 3469     Register dst_reg = as_Register($dst$$reg);
 3470     __ mov(dst_reg, (uint64_t)1);
 3471   %}
 3472 
 3473   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3474     C2_MacroAssembler _masm(&cbuf);
 3475     __ load_byte_map_base($dst$$Register);
 3476   %}
 3477 
 3478   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480     Register dst_reg = as_Register($dst$$reg);
 3481     address con = (address)$src$$constant;
 3482     if (con == NULL) {
 3483       ShouldNotReachHere();
 3484     } else {
 3485       relocInfo::relocType rtype = $src->constant_reloc();
 3486       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3487       __ set_narrow_oop(dst_reg, (jobject)con);
 3488     }
 3489   %}
 3490 
 3491   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3492     C2_MacroAssembler _masm(&cbuf);
 3493     Register dst_reg = as_Register($dst$$reg);
 3494     __ mov(dst_reg, zr);
 3495   %}
 3496 
 3497   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3498     C2_MacroAssembler _masm(&cbuf);
 3499     Register dst_reg = as_Register($dst$$reg);
 3500     address con = (address)$src$$constant;
 3501     if (con == NULL) {
 3502       ShouldNotReachHere();
 3503     } else {
 3504       relocInfo::relocType rtype = $src->constant_reloc();
 3505       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3506       __ set_narrow_klass(dst_reg, (Klass *)con);
 3507     }
 3508   %}
 3509 
 3510   // arithmetic encodings
 3511 
 3512   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3513     C2_MacroAssembler _masm(&cbuf);
 3514     Register dst_reg = as_Register($dst$$reg);
 3515     Register src_reg = as_Register($src1$$reg);
 3516     int32_t con = (int32_t)$src2$$constant;
 3517     // add has primary == 0, subtract has primary == 1
 3518     if ($primary) { con = -con; }
 3519     if (con < 0) {
 3520       __ subw(dst_reg, src_reg, -con);
 3521     } else {
 3522       __ addw(dst_reg, src_reg, con);
 3523     }
 3524   %}
 3525 
 3526   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3527     C2_MacroAssembler _masm(&cbuf);
 3528     Register dst_reg = as_Register($dst$$reg);
 3529     Register src_reg = as_Register($src1$$reg);
 3530     int32_t con = (int32_t)$src2$$constant;
 3531     // add has primary == 0, subtract has primary == 1
 3532     if ($primary) { con = -con; }
 3533     if (con < 0) {
 3534       __ sub(dst_reg, src_reg, -con);
 3535     } else {
 3536       __ add(dst_reg, src_reg, con);
 3537     }
 3538   %}
 3539 
 3540   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3541     C2_MacroAssembler _masm(&cbuf);
 3542    Register dst_reg = as_Register($dst$$reg);
 3543    Register src1_reg = as_Register($src1$$reg);
 3544    Register src2_reg = as_Register($src2$$reg);
 3545     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3546   %}
 3547 
 3548   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3549     C2_MacroAssembler _masm(&cbuf);
 3550    Register dst_reg = as_Register($dst$$reg);
 3551    Register src1_reg = as_Register($src1$$reg);
 3552    Register src2_reg = as_Register($src2$$reg);
 3553     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3554   %}
 3555 
 3556   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3557     C2_MacroAssembler _masm(&cbuf);
 3558    Register dst_reg = as_Register($dst$$reg);
 3559    Register src1_reg = as_Register($src1$$reg);
 3560    Register src2_reg = as_Register($src2$$reg);
 3561     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3562   %}
 3563 
 3564   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3565     C2_MacroAssembler _masm(&cbuf);
 3566    Register dst_reg = as_Register($dst$$reg);
 3567    Register src1_reg = as_Register($src1$$reg);
 3568    Register src2_reg = as_Register($src2$$reg);
 3569     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3570   %}
 3571 
 3572   // compare instruction encodings
 3573 
 3574   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3575     C2_MacroAssembler _masm(&cbuf);
 3576     Register reg1 = as_Register($src1$$reg);
 3577     Register reg2 = as_Register($src2$$reg);
 3578     __ cmpw(reg1, reg2);
 3579   %}
 3580 
 3581   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3582     C2_MacroAssembler _masm(&cbuf);
 3583     Register reg = as_Register($src1$$reg);
 3584     int32_t val = $src2$$constant;
 3585     if (val >= 0) {
 3586       __ subsw(zr, reg, val);
 3587     } else {
 3588       __ addsw(zr, reg, -val);
 3589     }
 3590   %}
 3591 
 3592   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3593     C2_MacroAssembler _masm(&cbuf);
 3594     Register reg1 = as_Register($src1$$reg);
 3595     uint32_t val = (uint32_t)$src2$$constant;
 3596     __ movw(rscratch1, val);
 3597     __ cmpw(reg1, rscratch1);
 3598   %}
 3599 
 3600   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3601     C2_MacroAssembler _masm(&cbuf);
 3602     Register reg1 = as_Register($src1$$reg);
 3603     Register reg2 = as_Register($src2$$reg);
 3604     __ cmp(reg1, reg2);
 3605   %}
 3606 
 3607   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3608     C2_MacroAssembler _masm(&cbuf);
 3609     Register reg = as_Register($src1$$reg);
 3610     int64_t val = $src2$$constant;
 3611     if (val >= 0) {
 3612       __ subs(zr, reg, val);
 3613     } else if (val != -val) {
 3614       __ adds(zr, reg, -val);
 3615     } else {
 3616     // aargh, Long.MIN_VALUE is a special case
 3617       __ orr(rscratch1, zr, (uint64_t)val);
 3618       __ subs(zr, reg, rscratch1);
 3619     }
 3620   %}
 3621 
 3622   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3623     C2_MacroAssembler _masm(&cbuf);
 3624     Register reg1 = as_Register($src1$$reg);
 3625     uint64_t val = (uint64_t)$src2$$constant;
 3626     __ mov(rscratch1, val);
 3627     __ cmp(reg1, rscratch1);
 3628   %}
 3629 
 3630   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3631     C2_MacroAssembler _masm(&cbuf);
 3632     Register reg1 = as_Register($src1$$reg);
 3633     Register reg2 = as_Register($src2$$reg);
 3634     __ cmp(reg1, reg2);
 3635   %}
 3636 
 3637   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3638     C2_MacroAssembler _masm(&cbuf);
 3639     Register reg1 = as_Register($src1$$reg);
 3640     Register reg2 = as_Register($src2$$reg);
 3641     __ cmpw(reg1, reg2);
 3642   %}
 3643 
 3644   enc_class aarch64_enc_testp(iRegP src) %{
 3645     C2_MacroAssembler _masm(&cbuf);
 3646     Register reg = as_Register($src$$reg);
 3647     __ cmp(reg, zr);
 3648   %}
 3649 
 3650   enc_class aarch64_enc_testn(iRegN src) %{
 3651     C2_MacroAssembler _masm(&cbuf);
 3652     Register reg = as_Register($src$$reg);
 3653     __ cmpw(reg, zr);
 3654   %}
 3655 
 3656   enc_class aarch64_enc_b(label lbl) %{
 3657     C2_MacroAssembler _masm(&cbuf);
 3658     Label *L = $lbl$$label;
 3659     __ b(*L);
 3660   %}
 3661 
 3662   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3663     C2_MacroAssembler _masm(&cbuf);
 3664     Label *L = $lbl$$label;
 3665     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3666   %}
 3667 
 3668   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3669     C2_MacroAssembler _masm(&cbuf);
 3670     Label *L = $lbl$$label;
 3671     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3672   %}
 3673 
 3674   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3675   %{
 3676      Register sub_reg = as_Register($sub$$reg);
 3677      Register super_reg = as_Register($super$$reg);
 3678      Register temp_reg = as_Register($temp$$reg);
 3679      Register result_reg = as_Register($result$$reg);
 3680 
 3681      Label miss;
 3682      C2_MacroAssembler _masm(&cbuf);
 3683      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3684                                      NULL, &miss,
 3685                                      /*set_cond_codes:*/ true);
 3686      if ($primary) {
 3687        __ mov(result_reg, zr);
 3688      }
 3689      __ bind(miss);
 3690   %}
 3691 
 3692   enc_class aarch64_enc_java_static_call(method meth) %{
 3693     C2_MacroAssembler _masm(&cbuf);
 3694 
 3695     address addr = (address)$meth$$method;
 3696     address call;
 3697     if (!_method) {
 3698       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3699       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3700       if (call == NULL) {
 3701         ciEnv::current()->record_failure("CodeCache is full");
 3702         return;
 3703       }
 3704     } else {
 3705       int method_index = resolved_method_index(cbuf);
 3706       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3707                                                   : static_call_Relocation::spec(method_index);
 3708       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3709       if (call == NULL) {
 3710         ciEnv::current()->record_failure("CodeCache is full");
 3711         return;
 3712       }
 3713       // Emit stub for static call
 3714       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3715       if (stub == NULL) {
 3716         ciEnv::current()->record_failure("CodeCache is full");
 3717         return;
 3718       }
 3719     }
 3720 
 3721     // Only non uncommon_trap calls need to reinitialize ptrue.
 3722     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3723       __ reinitialize_ptrue();
 3724     }
 3725   %}
 3726 
 3727   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3728     C2_MacroAssembler _masm(&cbuf);
 3729     int method_index = resolved_method_index(cbuf);
 3730     address call = __ ic_call((address)$meth$$method, method_index);
 3731     if (call == NULL) {
 3732       ciEnv::current()->record_failure("CodeCache is full");
 3733       return;
 3734     } else if (Compile::current()->max_vector_size() > 0) {
 3735       __ reinitialize_ptrue();
 3736     }
 3737   %}
 3738 
 3739   enc_class aarch64_enc_call_epilog() %{
 3740     C2_MacroAssembler _masm(&cbuf);
 3741     if (VerifyStackAtCalls) {
 3742       // Check that stack depth is unchanged: find majik cookie on stack
 3743       __ call_Unimplemented();
 3744     }
 3745   %}
 3746 
 3747   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3748     C2_MacroAssembler _masm(&cbuf);
 3749 
 3750     // some calls to generated routines (arraycopy code) are scheduled
 3751     // by C2 as runtime calls. if so we can call them using a br (they
 3752     // will be in a reachable segment) otherwise we have to use a blr
 3753     // which loads the absolute address into a register.
 3754     address entry = (address)$meth$$method;
 3755     CodeBlob *cb = CodeCache::find_blob(entry);
 3756     if (cb) {
 3757       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3758       if (call == NULL) {
 3759         ciEnv::current()->record_failure("CodeCache is full");
 3760         return;
 3761       }
 3762     } else {
 3763       Label retaddr;
 3764       __ adr(rscratch2, retaddr);
 3765       __ lea(rscratch1, RuntimeAddress(entry));
 3766       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3767       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3768       __ blr(rscratch1);
 3769       __ bind(retaddr);
 3770       __ add(sp, sp, 2 * wordSize);
 3771     }
 3772     if (Compile::current()->max_vector_size() > 0) {
 3773       __ reinitialize_ptrue();
 3774     }
 3775   %}
 3776 
 3777   enc_class aarch64_enc_rethrow() %{
 3778     C2_MacroAssembler _masm(&cbuf);
 3779     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3780   %}
 3781 
 3782   enc_class aarch64_enc_ret() %{
 3783     C2_MacroAssembler _masm(&cbuf);
 3784 #ifdef ASSERT
 3785     if (Compile::current()->max_vector_size() > 0) {
 3786       __ verify_ptrue();
 3787     }
 3788 #endif
 3789     __ ret(lr);
 3790   %}
 3791 
 3792   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3793     C2_MacroAssembler _masm(&cbuf);
 3794     Register target_reg = as_Register($jump_target$$reg);
 3795     __ br(target_reg);
 3796   %}
 3797 
 3798   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3799     C2_MacroAssembler _masm(&cbuf);
 3800     Register target_reg = as_Register($jump_target$$reg);
 3801     // exception oop should be in r0
 3802     // ret addr has been popped into lr
 3803     // callee expects it in r3
 3804     __ mov(r3, lr);
 3805     __ br(target_reg);
 3806   %}
 3807 
 3808   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3809     C2_MacroAssembler _masm(&cbuf);
 3810     Register oop = as_Register($object$$reg);
 3811     Register box = as_Register($box$$reg);
 3812     Register disp_hdr = as_Register($tmp$$reg);
 3813     Register tmp = as_Register($tmp2$$reg);
 3814     Label cont;
 3815     Label object_has_monitor;
 3816     Label cas_failed;
 3817 
 3818     assert_different_registers(oop, box, tmp, disp_hdr);
 3819 
 3820     // Load markWord from object into displaced_header.
 3821     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3822 
 3823     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3824       __ load_klass(tmp, oop);
 3825       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3826       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3827       __ br(Assembler::NE, cont);
 3828     }
 3829 
 3830     // Check for existing monitor
 3831     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3832 
 3833     // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3834     __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3835 
 3836     // Initialize the box. (Must happen before we update the object mark!)
 3837     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3838 
 3839     // Compare object markWord with an unlocked value (tmp) and if
 3840     // equal exchange the stack address of our box with object markWord.
 3841     // On failure disp_hdr contains the possibly locked markWord.
 3842     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3843                /*release*/ true, /*weak*/ false, disp_hdr);
 3844     __ br(Assembler::EQ, cont);
 3845 
 3846     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3847 
 3848     // If the compare-and-exchange succeeded, then we found an unlocked
 3849     // object, will have now locked it will continue at label cont
 3850 
 3851     __ bind(cas_failed);
 3852     // We did not see an unlocked object so try the fast recursive case.
 3853 
 3854     // Check if the owner is self by comparing the value in the
 3855     // markWord of object (disp_hdr) with the stack pointer.
 3856     __ mov(rscratch1, sp);
 3857     __ sub(disp_hdr, disp_hdr, rscratch1);
 3858     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3859     // If condition is true we are cont and hence we can store 0 as the
 3860     // displaced header in the box, which indicates that it is a recursive lock.
 3861     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3862     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3863 
 3864     __ b(cont);
 3865 
 3866     // Handle existing monitor.
 3867     __ bind(object_has_monitor);
 3868 
 3869     // The object's monitor m is unlocked iff m->owner == NULL,
 3870     // otherwise m->owner may contain a thread or a stack address.
 3871     //
 3872     // Try to CAS m->owner from NULL to current thread.
 3873     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3874     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3875                /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
 3876 
 3877     // Store a non-null value into the box to avoid looking like a re-entrant
 3878     // lock. The fast-path monitor unlock code checks for
 3879     // markWord::monitor_value so use markWord::unused_mark which has the
 3880     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3881     __ mov(tmp, (address)markWord::unused_mark().value());
 3882     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3883 
 3884     __ bind(cont);
 3885     // flag == EQ indicates success
 3886     // flag == NE indicates failure
 3887   %}
 3888 
 3889   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3890     C2_MacroAssembler _masm(&cbuf);
 3891     Register oop = as_Register($object$$reg);
 3892     Register box = as_Register($box$$reg);
 3893     Register disp_hdr = as_Register($tmp$$reg);
 3894     Register tmp = as_Register($tmp2$$reg);
 3895     Label cont;
 3896     Label object_has_monitor;
 3897 
 3898     assert_different_registers(oop, box, tmp, disp_hdr);
 3899 
 3900     // Find the lock address and load the displaced header from the stack.
 3901     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3902 
 3903     // If the displaced header is 0, we have a recursive unlock.
 3904     __ cmp(disp_hdr, zr);
 3905     __ br(Assembler::EQ, cont);
 3906 
 3907     // Handle existing monitor.
 3908     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3909     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3910 
 3911     // Check if it is still a light weight lock, this is is true if we
 3912     // see the stack address of the basicLock in the markWord of the
 3913     // object.
 3914 
 3915     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3916                /*release*/ true, /*weak*/ false, tmp);
 3917     __ b(cont);
 3918 
 3919     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3920 
 3921     // Handle existing monitor.
 3922     __ bind(object_has_monitor);
 3923     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3924     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3925     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3926     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3927     __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
 3928     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
 3929     __ cmp(rscratch1, zr); // Sets flags for result
 3930     __ br(Assembler::NE, cont);
 3931 
 3932     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3933     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3934     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3935     __ cmp(rscratch1, zr); // Sets flags for result
 3936     __ cbnz(rscratch1, cont);
 3937     // need a release store here
 3938     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3939     __ stlr(zr, tmp); // set unowned
 3940 
 3941     __ bind(cont);
 3942     // flag == EQ indicates success
 3943     // flag == NE indicates failure
 3944   %}
 3945 
 3946 %}
 3947 
 3948 //----------FRAME--------------------------------------------------------------
 3949 // Definition of frame structure and management information.
 3950 //
 3951 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3952 //                             |   (to get allocators register number
 3953 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3954 //  r   CALLER     |        |
 3955 //  o     |        +--------+      pad to even-align allocators stack-slot
 3956 //  w     V        |  pad0  |        numbers; owned by CALLER
 3957 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3958 //  h     ^        |   in   |  5
 3959 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3960 //  |     |        |        |  3
 3961 //  |     |        +--------+
 3962 //  V     |        | old out|      Empty on Intel, window on Sparc
 3963 //        |    old |preserve|      Must be even aligned.
 3964 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3965 //        |        |   in   |  3   area for Intel ret address
 3966 //     Owned by    |preserve|      Empty on Sparc.
 3967 //       SELF      +--------+
 3968 //        |        |  pad2  |  2   pad to align old SP
 3969 //        |        +--------+  1
 3970 //        |        | locks  |  0
 3971 //        |        +--------+----> OptoReg::stack0(), even aligned
 3972 //        |        |  pad1  | 11   pad to align new SP
 3973 //        |        +--------+
 3974 //        |        |        | 10
 3975 //        |        | spills |  9   spills
 3976 //        V        |        |  8   (pad0 slot for callee)
 3977 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3978 //        ^        |  out   |  7
 3979 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3980 //     Owned by    +--------+
 3981 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3982 //        |    new |preserve|      Must be even-aligned.
 3983 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3984 //        |        |        |
 3985 //
 3986 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3987 //         known from SELF's arguments and the Java calling convention.
 3988 //         Region 6-7 is determined per call site.
 3989 // Note 2: If the calling convention leaves holes in the incoming argument
 3990 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3991 //         are owned by the CALLEE.  Holes should not be nessecary in the
 3992 //         incoming area, as the Java calling convention is completely under
 3993 //         the control of the AD file.  Doubles can be sorted and packed to
 3994 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 3995 //         varargs C calling conventions.
 3996 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3997 //         even aligned with pad0 as needed.
 3998 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3999 //           (the latter is true on Intel but is it false on AArch64?)
 4000 //         region 6-11 is even aligned; it may be padded out more so that
 4001 //         the region from SP to FP meets the minimum stack alignment.
 4002 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4003 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4004 //         SP meets the minimum alignment.
 4005 
 4006 frame %{
 4007   // These three registers define part of the calling convention
 4008   // between compiled code and the interpreter.
 4009 
 4010   // Inline Cache Register or Method for I2C.
 4011   inline_cache_reg(R12);
 4012 
 4013   // Number of stack slots consumed by locking an object
 4014   sync_stack_slots(2);
 4015 
 4016   // Compiled code's Frame Pointer
 4017   frame_pointer(R31);
 4018 
 4019   // Interpreter stores its frame pointer in a register which is
 4020   // stored to the stack by I2CAdaptors.
 4021   // I2CAdaptors convert from interpreted java to compiled java.
 4022   interpreter_frame_pointer(R29);
 4023 
 4024   // Stack alignment requirement
 4025   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4026 
 4027   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4028   // for calls to C.  Supports the var-args backing area for register parms.
 4029   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4030 
 4031   // The after-PROLOG location of the return address.  Location of
 4032   // return address specifies a type (REG or STACK) and a number
 4033   // representing the register number (i.e. - use a register name) or
 4034   // stack slot.
 4035   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4036   // Otherwise, it is above the locks and verification slot and alignment word
 4037   // TODO this may well be correct but need to check why that - 2 is there
 4038   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4039   // which folds in the space used for monitors
 4040   return_addr(STACK - 2 +
 4041               align_up((Compile::current()->in_preserve_stack_slots() +
 4042                         Compile::current()->fixed_slots()),
 4043                        stack_alignment_in_slots()));
 4044 
 4045   // Location of compiled Java return values.  Same as C for now.
 4046   return_value
 4047   %{
 4048     // TODO do we allow ideal_reg == Op_RegN???
 4049     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4050            "only return normal values");
 4051 
 4052     static const int lo[Op_RegL + 1] = { // enum name
 4053       0,                                 // Op_Node
 4054       0,                                 // Op_Set
 4055       R0_num,                            // Op_RegN
 4056       R0_num,                            // Op_RegI
 4057       R0_num,                            // Op_RegP
 4058       V0_num,                            // Op_RegF
 4059       V0_num,                            // Op_RegD
 4060       R0_num                             // Op_RegL
 4061     };
 4062 
 4063     static const int hi[Op_RegL + 1] = { // enum name
 4064       0,                                 // Op_Node
 4065       0,                                 // Op_Set
 4066       OptoReg::Bad,                      // Op_RegN
 4067       OptoReg::Bad,                      // Op_RegI
 4068       R0_H_num,                          // Op_RegP
 4069       OptoReg::Bad,                      // Op_RegF
 4070       V0_H_num,                          // Op_RegD
 4071       R0_H_num                           // Op_RegL
 4072     };
 4073 
 4074     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4075   %}
 4076 %}
 4077 
 4078 //----------ATTRIBUTES---------------------------------------------------------
 4079 //----------Operand Attributes-------------------------------------------------
 4080 op_attrib op_cost(1);        // Required cost attribute
 4081 
 4082 //----------Instruction Attributes---------------------------------------------
 4083 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4084 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4085 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4086                                 // a non-matching short branch variant
 4087                                 // of some long branch?
 4088 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4089                                 // be a power of 2) specifies the
 4090                                 // alignment that some part of the
 4091                                 // instruction (not necessarily the
 4092                                 // start) requires.  If > 1, a
 4093                                 // compute_padding() function must be
 4094                                 // provided for the instruction
 4095 
 4096 //----------OPERANDS-----------------------------------------------------------
 4097 // Operand definitions must precede instruction definitions for correct parsing
 4098 // in the ADLC because operands constitute user defined types which are used in
 4099 // instruction definitions.
 4100 
 4101 //----------Simple Operands----------------------------------------------------
 4102 
 4103 // Integer operands 32 bit
 4104 // 32 bit immediate
 4105 operand immI()
 4106 %{
 4107   match(ConI);
 4108 
 4109   op_cost(0);
 4110   format %{ %}
 4111   interface(CONST_INTER);
 4112 %}
 4113 
 4114 // 32 bit zero
 4115 operand immI0()
 4116 %{
 4117   predicate(n->get_int() == 0);
 4118   match(ConI);
 4119 
 4120   op_cost(0);
 4121   format %{ %}
 4122   interface(CONST_INTER);
 4123 %}
 4124 
 4125 // 32 bit unit increment
 4126 operand immI_1()
 4127 %{
 4128   predicate(n->get_int() == 1);
 4129   match(ConI);
 4130 
 4131   op_cost(0);
 4132   format %{ %}
 4133   interface(CONST_INTER);
 4134 %}
 4135 
 4136 // 32 bit unit decrement
 4137 operand immI_M1()
 4138 %{
 4139   predicate(n->get_int() == -1);
 4140   match(ConI);
 4141 
 4142   op_cost(0);
 4143   format %{ %}
 4144   interface(CONST_INTER);
 4145 %}
 4146 
 4147 // Shift values for add/sub extension shift
 4148 operand immIExt()
 4149 %{
 4150   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4151   match(ConI);
 4152 
 4153   op_cost(0);
 4154   format %{ %}
 4155   interface(CONST_INTER);
 4156 %}
 4157 
 4158 operand immI_gt_1()
 4159 %{
 4160   predicate(n->get_int() > 1);
 4161   match(ConI);
 4162 
 4163   op_cost(0);
 4164   format %{ %}
 4165   interface(CONST_INTER);
 4166 %}
 4167 
 4168 operand immI_le_4()
 4169 %{
 4170   predicate(n->get_int() <= 4);
 4171   match(ConI);
 4172 
 4173   op_cost(0);
 4174   format %{ %}
 4175   interface(CONST_INTER);
 4176 %}
 4177 
 4178 operand immI_31()
 4179 %{
 4180   predicate(n->get_int() == 31);
 4181   match(ConI);
 4182 
 4183   op_cost(0);
 4184   format %{ %}
 4185   interface(CONST_INTER);
 4186 %}
 4187 
 4188 operand immI_2()
 4189 %{
 4190   predicate(n->get_int() == 2);
 4191   match(ConI);
 4192 
 4193   op_cost(0);
 4194   format %{ %}
 4195   interface(CONST_INTER);
 4196 %}
 4197 
 4198 operand immI_4()
 4199 %{
 4200   predicate(n->get_int() == 4);
 4201   match(ConI);
 4202 
 4203   op_cost(0);
 4204   format %{ %}
 4205   interface(CONST_INTER);
 4206 %}
 4207 
 4208 operand immI_8()
 4209 %{
 4210   predicate(n->get_int() == 8);
 4211   match(ConI);
 4212 
 4213   op_cost(0);
 4214   format %{ %}
 4215   interface(CONST_INTER);
 4216 %}
 4217 
 4218 operand immI_16()
 4219 %{
 4220   predicate(n->get_int() == 16);
 4221   match(ConI);
 4222 
 4223   op_cost(0);
 4224   format %{ %}
 4225   interface(CONST_INTER);
 4226 %}
 4227 
 4228 operand immI_24()
 4229 %{
 4230   predicate(n->get_int() == 24);
 4231   match(ConI);
 4232 
 4233   op_cost(0);
 4234   format %{ %}
 4235   interface(CONST_INTER);
 4236 %}
 4237 
 4238 operand immI_32()
 4239 %{
 4240   predicate(n->get_int() == 32);
 4241   match(ConI);
 4242 
 4243   op_cost(0);
 4244   format %{ %}
 4245   interface(CONST_INTER);
 4246 %}
 4247 
 4248 operand immI_48()
 4249 %{
 4250   predicate(n->get_int() == 48);
 4251   match(ConI);
 4252 
 4253   op_cost(0);
 4254   format %{ %}
 4255   interface(CONST_INTER);
 4256 %}
 4257 
 4258 operand immI_56()
 4259 %{
 4260   predicate(n->get_int() == 56);
 4261   match(ConI);
 4262 
 4263   op_cost(0);
 4264   format %{ %}
 4265   interface(CONST_INTER);
 4266 %}
 4267 
 4268 operand immI_63()
 4269 %{
 4270   predicate(n->get_int() == 63);
 4271   match(ConI);
 4272 
 4273   op_cost(0);
 4274   format %{ %}
 4275   interface(CONST_INTER);
 4276 %}
 4277 
 4278 operand immI_64()
 4279 %{
 4280   predicate(n->get_int() == 64);
 4281   match(ConI);
 4282 
 4283   op_cost(0);
 4284   format %{ %}
 4285   interface(CONST_INTER);
 4286 %}
 4287 
 4288 operand immI_255()
 4289 %{
 4290   predicate(n->get_int() == 255);
 4291   match(ConI);
 4292 
 4293   op_cost(0);
 4294   format %{ %}
 4295   interface(CONST_INTER);
 4296 %}
 4297 
 4298 operand immI_65535()
 4299 %{
 4300   predicate(n->get_int() == 65535);
 4301   match(ConI);
 4302 
 4303   op_cost(0);
 4304   format %{ %}
 4305   interface(CONST_INTER);
 4306 %}
 4307 
 4308 operand immL_255()
 4309 %{
 4310   predicate(n->get_long() == 255L);
 4311   match(ConL);
 4312 
 4313   op_cost(0);
 4314   format %{ %}
 4315   interface(CONST_INTER);
 4316 %}
 4317 
 4318 operand immL_65535()
 4319 %{
 4320   predicate(n->get_long() == 65535L);
 4321   match(ConL);
 4322 
 4323   op_cost(0);
 4324   format %{ %}
 4325   interface(CONST_INTER);
 4326 %}
 4327 
 4328 operand immL_4294967295()
 4329 %{
 4330   predicate(n->get_long() == 4294967295L);
 4331   match(ConL);
 4332 
 4333   op_cost(0);
 4334   format %{ %}
 4335   interface(CONST_INTER);
 4336 %}
 4337 
 4338 operand immL_bitmask()
 4339 %{
 4340   predicate((n->get_long() != 0)
 4341             && ((n->get_long() & 0xc000000000000000l) == 0)
 4342             && is_power_of_2(n->get_long() + 1));
 4343   match(ConL);
 4344 
 4345   op_cost(0);
 4346   format %{ %}
 4347   interface(CONST_INTER);
 4348 %}
 4349 
 4350 operand immI_bitmask()
 4351 %{
 4352   predicate((n->get_int() != 0)
 4353             && ((n->get_int() & 0xc0000000) == 0)
 4354             && is_power_of_2(n->get_int() + 1));
 4355   match(ConI);
 4356 
 4357   op_cost(0);
 4358   format %{ %}
 4359   interface(CONST_INTER);
 4360 %}
 4361 
 4362 operand immL_positive_bitmaskI()
 4363 %{
 4364   predicate((n->get_long() != 0)
 4365             && ((julong)n->get_long() < 0x80000000ULL)
 4366             && is_power_of_2(n->get_long() + 1));
 4367   match(ConL);
 4368 
 4369   op_cost(0);
 4370   format %{ %}
 4371   interface(CONST_INTER);
 4372 %}
 4373 
 4374 // Scale values for scaled offset addressing modes (up to long but not quad)
 4375 operand immIScale()
 4376 %{
 4377   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4378   match(ConI);
 4379 
 4380   op_cost(0);
 4381   format %{ %}
 4382   interface(CONST_INTER);
 4383 %}
 4384 
 4385 // 26 bit signed offset -- for pc-relative branches
 4386 operand immI26()
 4387 %{
 4388   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4389   match(ConI);
 4390 
 4391   op_cost(0);
 4392   format %{ %}
 4393   interface(CONST_INTER);
 4394 %}
 4395 
 4396 // 19 bit signed offset -- for pc-relative loads
 4397 operand immI19()
 4398 %{
 4399   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4400   match(ConI);
 4401 
 4402   op_cost(0);
 4403   format %{ %}
 4404   interface(CONST_INTER);
 4405 %}
 4406 
 4407 // 12 bit unsigned offset -- for base plus immediate loads
 4408 operand immIU12()
 4409 %{
 4410   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4411   match(ConI);
 4412 
 4413   op_cost(0);
 4414   format %{ %}
 4415   interface(CONST_INTER);
 4416 %}
 4417 
 4418 operand immLU12()
 4419 %{
 4420   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4421   match(ConL);
 4422 
 4423   op_cost(0);
 4424   format %{ %}
 4425   interface(CONST_INTER);
 4426 %}
 4427 
 4428 // Offset for scaled or unscaled immediate loads and stores
 4429 operand immIOffset()
 4430 %{
 4431   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4432   match(ConI);
 4433 
 4434   op_cost(0);
 4435   format %{ %}
 4436   interface(CONST_INTER);
 4437 %}
 4438 
 4439 operand immIOffset1()
 4440 %{
 4441   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4442   match(ConI);
 4443 
 4444   op_cost(0);
 4445   format %{ %}
 4446   interface(CONST_INTER);
 4447 %}
 4448 
 4449 operand immIOffset2()
 4450 %{
 4451   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4452   match(ConI);
 4453 
 4454   op_cost(0);
 4455   format %{ %}
 4456   interface(CONST_INTER);
 4457 %}
 4458 
 4459 operand immIOffset4()
 4460 %{
 4461   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4462   match(ConI);
 4463 
 4464   op_cost(0);
 4465   format %{ %}
 4466   interface(CONST_INTER);
 4467 %}
 4468 
 4469 operand immIOffset8()
 4470 %{
 4471   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4472   match(ConI);
 4473 
 4474   op_cost(0);
 4475   format %{ %}
 4476   interface(CONST_INTER);
 4477 %}
 4478 
 4479 operand immIOffset16()
 4480 %{
 4481   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4482   match(ConI);
 4483 
 4484   op_cost(0);
 4485   format %{ %}
 4486   interface(CONST_INTER);
 4487 %}
 4488 
 4489 operand immLoffset()
 4490 %{
 4491   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4492   match(ConL);
 4493 
 4494   op_cost(0);
 4495   format %{ %}
 4496   interface(CONST_INTER);
 4497 %}
 4498 
 4499 operand immLoffset1()
 4500 %{
 4501   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4502   match(ConL);
 4503 
 4504   op_cost(0);
 4505   format %{ %}
 4506   interface(CONST_INTER);
 4507 %}
 4508 
 4509 operand immLoffset2()
 4510 %{
 4511   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4512   match(ConL);
 4513 
 4514   op_cost(0);
 4515   format %{ %}
 4516   interface(CONST_INTER);
 4517 %}
 4518 
 4519 operand immLoffset4()
 4520 %{
 4521   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4522   match(ConL);
 4523 
 4524   op_cost(0);
 4525   format %{ %}
 4526   interface(CONST_INTER);
 4527 %}
 4528 
 4529 operand immLoffset8()
 4530 %{
 4531   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4532   match(ConL);
 4533 
 4534   op_cost(0);
 4535   format %{ %}
 4536   interface(CONST_INTER);
 4537 %}
 4538 
 4539 operand immLoffset16()
 4540 %{
 4541   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4542   match(ConL);
 4543 
 4544   op_cost(0);
 4545   format %{ %}
 4546   interface(CONST_INTER);
 4547 %}
 4548 
 4549 // 8 bit signed value.
 4550 operand immI8()
 4551 %{
 4552   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4553   match(ConI);
 4554 
 4555   op_cost(0);
 4556   format %{ %}
 4557   interface(CONST_INTER);
 4558 %}
 4559 
 4560 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4561 operand immI8_shift8()
 4562 %{
 4563   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4564             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4565   match(ConI);
 4566 
 4567   op_cost(0);
 4568   format %{ %}
 4569   interface(CONST_INTER);
 4570 %}
 4571 
 4572 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4573 operand immL8_shift8()
 4574 %{
 4575   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4576             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4577   match(ConL);
 4578 
 4579   op_cost(0);
 4580   format %{ %}
 4581   interface(CONST_INTER);
 4582 %}
 4583 
 4584 // 32 bit integer valid for add sub immediate
 4585 operand immIAddSub()
 4586 %{
 4587   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4588   match(ConI);
 4589   op_cost(0);
 4590   format %{ %}
 4591   interface(CONST_INTER);
 4592 %}
 4593 
 4594 // 32 bit unsigned integer valid for logical immediate
 4595 // TODO -- check this is right when e.g the mask is 0x80000000
 4596 operand immILog()
 4597 %{
 4598   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4599   match(ConI);
 4600 
 4601   op_cost(0);
 4602   format %{ %}
 4603   interface(CONST_INTER);
 4604 %}
 4605 
 4606 // Integer operands 64 bit
 4607 // 64 bit immediate
 4608 operand immL()
 4609 %{
 4610   match(ConL);
 4611 
 4612   op_cost(0);
 4613   format %{ %}
 4614   interface(CONST_INTER);
 4615 %}
 4616 
 4617 // 64 bit zero
 4618 operand immL0()
 4619 %{
 4620   predicate(n->get_long() == 0);
 4621   match(ConL);
 4622 
 4623   op_cost(0);
 4624   format %{ %}
 4625   interface(CONST_INTER);
 4626 %}
 4627 
 4628 // 64 bit unit increment
 4629 operand immL_1()
 4630 %{
 4631   predicate(n->get_long() == 1);
 4632   match(ConL);
 4633 
 4634   op_cost(0);
 4635   format %{ %}
 4636   interface(CONST_INTER);
 4637 %}
 4638 
 4639 // 64 bit unit decrement
 4640 operand immL_M1()
 4641 %{
 4642   predicate(n->get_long() == -1);
 4643   match(ConL);
 4644 
 4645   op_cost(0);
 4646   format %{ %}
 4647   interface(CONST_INTER);
 4648 %}
 4649 
 4650 // 32 bit offset of pc in thread anchor
 4651 
 4652 operand immL_pc_off()
 4653 %{
 4654   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4655                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4656   match(ConL);
 4657 
 4658   op_cost(0);
 4659   format %{ %}
 4660   interface(CONST_INTER);
 4661 %}
 4662 
 4663 // 64 bit integer valid for add sub immediate
 4664 operand immLAddSub()
 4665 %{
 4666   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4667   match(ConL);
 4668   op_cost(0);
 4669   format %{ %}
 4670   interface(CONST_INTER);
 4671 %}
 4672 
 4673 // 64 bit integer valid for logical immediate
 4674 operand immLLog()
 4675 %{
 4676   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4677   match(ConL);
 4678   op_cost(0);
 4679   format %{ %}
 4680   interface(CONST_INTER);
 4681 %}
 4682 
 4683 // Long Immediate: low 32-bit mask
 4684 operand immL_32bits()
 4685 %{
 4686   predicate(n->get_long() == 0xFFFFFFFFL);
 4687   match(ConL);
 4688   op_cost(0);
 4689   format %{ %}
 4690   interface(CONST_INTER);
 4691 %}
 4692 
 4693 // Pointer operands
 4694 // Pointer Immediate
 4695 operand immP()
 4696 %{
 4697   match(ConP);
 4698 
 4699   op_cost(0);
 4700   format %{ %}
 4701   interface(CONST_INTER);
 4702 %}
 4703 
 4704 // NULL Pointer Immediate
 4705 operand immP0()
 4706 %{
 4707   predicate(n->get_ptr() == 0);
 4708   match(ConP);
 4709 
 4710   op_cost(0);
 4711   format %{ %}
 4712   interface(CONST_INTER);
 4713 %}
 4714 
 4715 // Pointer Immediate One
 4716 // this is used in object initialization (initial object header)
 4717 operand immP_1()
 4718 %{
 4719   predicate(n->get_ptr() == 1);
 4720   match(ConP);
 4721 
 4722   op_cost(0);
 4723   format %{ %}
 4724   interface(CONST_INTER);
 4725 %}
 4726 
 4727 // Card Table Byte Map Base
 4728 operand immByteMapBase()
 4729 %{
 4730   // Get base of card map
 4731   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4732             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4733   match(ConP);
 4734 
 4735   op_cost(0);
 4736   format %{ %}
 4737   interface(CONST_INTER);
 4738 %}
 4739 
 4740 // Pointer Immediate Minus One
 4741 // this is used when we want to write the current PC to the thread anchor
 4742 operand immP_M1()
 4743 %{
 4744   predicate(n->get_ptr() == -1);
 4745   match(ConP);
 4746 
 4747   op_cost(0);
 4748   format %{ %}
 4749   interface(CONST_INTER);
 4750 %}
 4751 
 4752 // Pointer Immediate Minus Two
 4753 // this is used when we want to write the current PC to the thread anchor
 4754 operand immP_M2()
 4755 %{
 4756   predicate(n->get_ptr() == -2);
 4757   match(ConP);
 4758 
 4759   op_cost(0);
 4760   format %{ %}
 4761   interface(CONST_INTER);
 4762 %}
 4763 
 4764 // Float and Double operands
 4765 // Double Immediate
 4766 operand immD()
 4767 %{
 4768   match(ConD);
 4769   op_cost(0);
 4770   format %{ %}
 4771   interface(CONST_INTER);
 4772 %}
 4773 
 4774 // Double Immediate: +0.0d
 4775 operand immD0()
 4776 %{
 4777   predicate(jlong_cast(n->getd()) == 0);
 4778   match(ConD);
 4779 
 4780   op_cost(0);
 4781   format %{ %}
 4782   interface(CONST_INTER);
 4783 %}
 4784 
 4785 // constant 'double +0.0'.
 4786 operand immDPacked()
 4787 %{
 4788   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4789   match(ConD);
 4790   op_cost(0);
 4791   format %{ %}
 4792   interface(CONST_INTER);
 4793 %}
 4794 
 4795 // Float Immediate
 4796 operand immF()
 4797 %{
 4798   match(ConF);
 4799   op_cost(0);
 4800   format %{ %}
 4801   interface(CONST_INTER);
 4802 %}
 4803 
 4804 // Float Immediate: +0.0f.
 4805 operand immF0()
 4806 %{
 4807   predicate(jint_cast(n->getf()) == 0);
 4808   match(ConF);
 4809 
 4810   op_cost(0);
 4811   format %{ %}
 4812   interface(CONST_INTER);
 4813 %}
 4814 
 4815 //
 4816 operand immFPacked()
 4817 %{
 4818   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4819   match(ConF);
 4820   op_cost(0);
 4821   format %{ %}
 4822   interface(CONST_INTER);
 4823 %}
 4824 
 4825 // Narrow pointer operands
 4826 // Narrow Pointer Immediate
 4827 operand immN()
 4828 %{
 4829   match(ConN);
 4830 
 4831   op_cost(0);
 4832   format %{ %}
 4833   interface(CONST_INTER);
 4834 %}
 4835 
 4836 // Narrow NULL Pointer Immediate
 4837 operand immN0()
 4838 %{
 4839   predicate(n->get_narrowcon() == 0);
 4840   match(ConN);
 4841 
 4842   op_cost(0);
 4843   format %{ %}
 4844   interface(CONST_INTER);
 4845 %}
 4846 
 4847 operand immNKlass()
 4848 %{
 4849   match(ConNKlass);
 4850 
 4851   op_cost(0);
 4852   format %{ %}
 4853   interface(CONST_INTER);
 4854 %}
 4855 
 4856 // Integer 32 bit Register Operands
 4857 // Integer 32 bitRegister (excludes SP)
 4858 operand iRegI()
 4859 %{
 4860   constraint(ALLOC_IN_RC(any_reg32));
 4861   match(RegI);
 4862   match(iRegINoSp);
 4863   op_cost(0);
 4864   format %{ %}
 4865   interface(REG_INTER);
 4866 %}
 4867 
 4868 // Integer 32 bit Register not Special
 4869 operand iRegINoSp()
 4870 %{
 4871   constraint(ALLOC_IN_RC(no_special_reg32));
 4872   match(RegI);
 4873   op_cost(0);
 4874   format %{ %}
 4875   interface(REG_INTER);
 4876 %}
 4877 
 4878 // Integer 64 bit Register Operands
 4879 // Integer 64 bit Register (includes SP)
 4880 operand iRegL()
 4881 %{
 4882   constraint(ALLOC_IN_RC(any_reg));
 4883   match(RegL);
 4884   match(iRegLNoSp);
 4885   op_cost(0);
 4886   format %{ %}
 4887   interface(REG_INTER);
 4888 %}
 4889 
 4890 // Integer 64 bit Register not Special
 4891 operand iRegLNoSp()
 4892 %{
 4893   constraint(ALLOC_IN_RC(no_special_reg));
 4894   match(RegL);
 4895   match(iRegL_R0);
 4896   format %{ %}
 4897   interface(REG_INTER);
 4898 %}
 4899 
 4900 // Pointer Register Operands
 4901 // Pointer Register
 4902 operand iRegP()
 4903 %{
 4904   constraint(ALLOC_IN_RC(ptr_reg));
 4905   match(RegP);
 4906   match(iRegPNoSp);
 4907   match(iRegP_R0);
 4908   //match(iRegP_R2);
 4909   //match(iRegP_R4);
 4910   //match(iRegP_R5);
 4911   match(thread_RegP);
 4912   op_cost(0);
 4913   format %{ %}
 4914   interface(REG_INTER);
 4915 %}
 4916 
 4917 // Pointer 64 bit Register not Special
 4918 operand iRegPNoSp()
 4919 %{
 4920   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4921   match(RegP);
 4922   // match(iRegP);
 4923   // match(iRegP_R0);
 4924   // match(iRegP_R2);
 4925   // match(iRegP_R4);
 4926   // match(iRegP_R5);
 4927   // match(thread_RegP);
 4928   op_cost(0);
 4929   format %{ %}
 4930   interface(REG_INTER);
 4931 %}
 4932 
 4933 // Pointer 64 bit Register R0 only
 4934 operand iRegP_R0()
 4935 %{
 4936   constraint(ALLOC_IN_RC(r0_reg));
 4937   match(RegP);
 4938   // match(iRegP);
 4939   match(iRegPNoSp);
 4940   op_cost(0);
 4941   format %{ %}
 4942   interface(REG_INTER);
 4943 %}
 4944 
 4945 // Pointer 64 bit Register R1 only
 4946 operand iRegP_R1()
 4947 %{
 4948   constraint(ALLOC_IN_RC(r1_reg));
 4949   match(RegP);
 4950   // match(iRegP);
 4951   match(iRegPNoSp);
 4952   op_cost(0);
 4953   format %{ %}
 4954   interface(REG_INTER);
 4955 %}
 4956 
 4957 // Pointer 64 bit Register R2 only
 4958 operand iRegP_R2()
 4959 %{
 4960   constraint(ALLOC_IN_RC(r2_reg));
 4961   match(RegP);
 4962   // match(iRegP);
 4963   match(iRegPNoSp);
 4964   op_cost(0);
 4965   format %{ %}
 4966   interface(REG_INTER);
 4967 %}
 4968 
 4969 // Pointer 64 bit Register R3 only
 4970 operand iRegP_R3()
 4971 %{
 4972   constraint(ALLOC_IN_RC(r3_reg));
 4973   match(RegP);
 4974   // match(iRegP);
 4975   match(iRegPNoSp);
 4976   op_cost(0);
 4977   format %{ %}
 4978   interface(REG_INTER);
 4979 %}
 4980 
 4981 // Pointer 64 bit Register R4 only
 4982 operand iRegP_R4()
 4983 %{
 4984   constraint(ALLOC_IN_RC(r4_reg));
 4985   match(RegP);
 4986   // match(iRegP);
 4987   match(iRegPNoSp);
 4988   op_cost(0);
 4989   format %{ %}
 4990   interface(REG_INTER);
 4991 %}
 4992 
 4993 // Pointer 64 bit Register R5 only
 4994 operand iRegP_R5()
 4995 %{
 4996   constraint(ALLOC_IN_RC(r5_reg));
 4997   match(RegP);
 4998   // match(iRegP);
 4999   match(iRegPNoSp);
 5000   op_cost(0);
 5001   format %{ %}
 5002   interface(REG_INTER);
 5003 %}
 5004 
 5005 // Pointer 64 bit Register R10 only
 5006 operand iRegP_R10()
 5007 %{
 5008   constraint(ALLOC_IN_RC(r10_reg));
 5009   match(RegP);
 5010   // match(iRegP);
 5011   match(iRegPNoSp);
 5012   op_cost(0);
 5013   format %{ %}
 5014   interface(REG_INTER);
 5015 %}
 5016 
 5017 // Long 64 bit Register R0 only
 5018 operand iRegL_R0()
 5019 %{
 5020   constraint(ALLOC_IN_RC(r0_reg));
 5021   match(RegL);
 5022   match(iRegLNoSp);
 5023   op_cost(0);
 5024   format %{ %}
 5025   interface(REG_INTER);
 5026 %}
 5027 
 5028 // Long 64 bit Register R2 only
 5029 operand iRegL_R2()
 5030 %{
 5031   constraint(ALLOC_IN_RC(r2_reg));
 5032   match(RegL);
 5033   match(iRegLNoSp);
 5034   op_cost(0);
 5035   format %{ %}
 5036   interface(REG_INTER);
 5037 %}
 5038 
 5039 // Long 64 bit Register R3 only
 5040 operand iRegL_R3()
 5041 %{
 5042   constraint(ALLOC_IN_RC(r3_reg));
 5043   match(RegL);
 5044   match(iRegLNoSp);
 5045   op_cost(0);
 5046   format %{ %}
 5047   interface(REG_INTER);
 5048 %}
 5049 
 5050 // Long 64 bit Register R11 only
 5051 operand iRegL_R11()
 5052 %{
 5053   constraint(ALLOC_IN_RC(r11_reg));
 5054   match(RegL);
 5055   match(iRegLNoSp);
 5056   op_cost(0);
 5057   format %{ %}
 5058   interface(REG_INTER);
 5059 %}
 5060 
 5061 // Pointer 64 bit Register FP only
 5062 operand iRegP_FP()
 5063 %{
 5064   constraint(ALLOC_IN_RC(fp_reg));
 5065   match(RegP);
 5066   // match(iRegP);
 5067   op_cost(0);
 5068   format %{ %}
 5069   interface(REG_INTER);
 5070 %}
 5071 
 5072 // Register R0 only
 5073 operand iRegI_R0()
 5074 %{
 5075   constraint(ALLOC_IN_RC(int_r0_reg));
 5076   match(RegI);
 5077   match(iRegINoSp);
 5078   op_cost(0);
 5079   format %{ %}
 5080   interface(REG_INTER);
 5081 %}
 5082 
 5083 // Register R2 only
 5084 operand iRegI_R2()
 5085 %{
 5086   constraint(ALLOC_IN_RC(int_r2_reg));
 5087   match(RegI);
 5088   match(iRegINoSp);
 5089   op_cost(0);
 5090   format %{ %}
 5091   interface(REG_INTER);
 5092 %}
 5093 
 5094 // Register R3 only
 5095 operand iRegI_R3()
 5096 %{
 5097   constraint(ALLOC_IN_RC(int_r3_reg));
 5098   match(RegI);
 5099   match(iRegINoSp);
 5100   op_cost(0);
 5101   format %{ %}
 5102   interface(REG_INTER);
 5103 %}
 5104 
 5105 
 5106 // Register R4 only
 5107 operand iRegI_R4()
 5108 %{
 5109   constraint(ALLOC_IN_RC(int_r4_reg));
 5110   match(RegI);
 5111   match(iRegINoSp);
 5112   op_cost(0);
 5113   format %{ %}
 5114   interface(REG_INTER);
 5115 %}
 5116 
 5117 
 5118 // Pointer Register Operands
 5119 // Narrow Pointer Register
 5120 operand iRegN()
 5121 %{
 5122   constraint(ALLOC_IN_RC(any_reg32));
 5123   match(RegN);
 5124   match(iRegNNoSp);
 5125   op_cost(0);
 5126   format %{ %}
 5127   interface(REG_INTER);
 5128 %}
 5129 
 5130 operand iRegN_R0()
 5131 %{
 5132   constraint(ALLOC_IN_RC(r0_reg));
 5133   match(iRegN);
 5134   op_cost(0);
 5135   format %{ %}
 5136   interface(REG_INTER);
 5137 %}
 5138 
 5139 operand iRegN_R2()
 5140 %{
 5141   constraint(ALLOC_IN_RC(r2_reg));
 5142   match(iRegN);
 5143   op_cost(0);
 5144   format %{ %}
 5145   interface(REG_INTER);
 5146 %}
 5147 
 5148 operand iRegN_R3()
 5149 %{
 5150   constraint(ALLOC_IN_RC(r3_reg));
 5151   match(iRegN);
 5152   op_cost(0);
 5153   format %{ %}
 5154   interface(REG_INTER);
 5155 %}
 5156 
 5157 // Integer 64 bit Register not Special
 5158 operand iRegNNoSp()
 5159 %{
 5160   constraint(ALLOC_IN_RC(no_special_reg32));
 5161   match(RegN);
 5162   op_cost(0);
 5163   format %{ %}
 5164   interface(REG_INTER);
 5165 %}
 5166 
 5167 // heap base register -- used for encoding immN0
 5168 
 5169 operand iRegIHeapbase()
 5170 %{
 5171   constraint(ALLOC_IN_RC(heapbase_reg));
 5172   match(RegI);
 5173   op_cost(0);
 5174   format %{ %}
 5175   interface(REG_INTER);
 5176 %}
 5177 
 5178 // Float Register
 5179 // Float register operands
 5180 operand vRegF()
 5181 %{
 5182   constraint(ALLOC_IN_RC(float_reg));
 5183   match(RegF);
 5184 
 5185   op_cost(0);
 5186   format %{ %}
 5187   interface(REG_INTER);
 5188 %}
 5189 
 5190 // Double Register
 5191 // Double register operands
 5192 operand vRegD()
 5193 %{
 5194   constraint(ALLOC_IN_RC(double_reg));
 5195   match(RegD);
 5196 
 5197   op_cost(0);
 5198   format %{ %}
 5199   interface(REG_INTER);
 5200 %}
 5201 
 5202 // Generic vector class. This will be used for
 5203 // all vector operands, including NEON and SVE,
 5204 // but currently only used for SVE VecA.
 5205 operand vReg()
 5206 %{
 5207   constraint(ALLOC_IN_RC(vectora_reg));
 5208   match(VecA);
 5209   op_cost(0);
 5210   format %{ %}
 5211   interface(REG_INTER);
 5212 %}
 5213 
 5214 operand vecD()
 5215 %{
 5216   constraint(ALLOC_IN_RC(vectord_reg));
 5217   match(VecD);
 5218 
 5219   op_cost(0);
 5220   format %{ %}
 5221   interface(REG_INTER);
 5222 %}
 5223 
 5224 operand vecX()
 5225 %{
 5226   constraint(ALLOC_IN_RC(vectorx_reg));
 5227   match(VecX);
 5228 
 5229   op_cost(0);
 5230   format %{ %}
 5231   interface(REG_INTER);
 5232 %}
 5233 
 5234 operand vRegD_V0()
 5235 %{
 5236   constraint(ALLOC_IN_RC(v0_reg));
 5237   match(RegD);
 5238   op_cost(0);
 5239   format %{ %}
 5240   interface(REG_INTER);
 5241 %}
 5242 
 5243 operand vRegD_V1()
 5244 %{
 5245   constraint(ALLOC_IN_RC(v1_reg));
 5246   match(RegD);
 5247   op_cost(0);
 5248   format %{ %}
 5249   interface(REG_INTER);
 5250 %}
 5251 
 5252 operand vRegD_V2()
 5253 %{
 5254   constraint(ALLOC_IN_RC(v2_reg));
 5255   match(RegD);
 5256   op_cost(0);
 5257   format %{ %}
 5258   interface(REG_INTER);
 5259 %}
 5260 
 5261 operand vRegD_V3()
 5262 %{
 5263   constraint(ALLOC_IN_RC(v3_reg));
 5264   match(RegD);
 5265   op_cost(0);
 5266   format %{ %}
 5267   interface(REG_INTER);
 5268 %}
 5269 
 5270 operand vRegD_V4()
 5271 %{
 5272   constraint(ALLOC_IN_RC(v4_reg));
 5273   match(RegD);
 5274   op_cost(0);
 5275   format %{ %}
 5276   interface(REG_INTER);
 5277 %}
 5278 
 5279 operand vRegD_V5()
 5280 %{
 5281   constraint(ALLOC_IN_RC(v5_reg));
 5282   match(RegD);
 5283   op_cost(0);
 5284   format %{ %}
 5285   interface(REG_INTER);
 5286 %}
 5287 
 5288 operand vRegD_V6()
 5289 %{
 5290   constraint(ALLOC_IN_RC(v6_reg));
 5291   match(RegD);
 5292   op_cost(0);
 5293   format %{ %}
 5294   interface(REG_INTER);
 5295 %}
 5296 
 5297 operand vRegD_V7()
 5298 %{
 5299   constraint(ALLOC_IN_RC(v7_reg));
 5300   match(RegD);
 5301   op_cost(0);
 5302   format %{ %}
 5303   interface(REG_INTER);
 5304 %}
 5305 
 5306 operand vRegD_V8()
 5307 %{
 5308   constraint(ALLOC_IN_RC(v8_reg));
 5309   match(RegD);
 5310   op_cost(0);
 5311   format %{ %}
 5312   interface(REG_INTER);
 5313 %}
 5314 
 5315 operand vRegD_V9()
 5316 %{
 5317   constraint(ALLOC_IN_RC(v9_reg));
 5318   match(RegD);
 5319   op_cost(0);
 5320   format %{ %}
 5321   interface(REG_INTER);
 5322 %}
 5323 
 5324 operand vRegD_V10()
 5325 %{
 5326   constraint(ALLOC_IN_RC(v10_reg));
 5327   match(RegD);
 5328   op_cost(0);
 5329   format %{ %}
 5330   interface(REG_INTER);
 5331 %}
 5332 
 5333 operand vRegD_V11()
 5334 %{
 5335   constraint(ALLOC_IN_RC(v11_reg));
 5336   match(RegD);
 5337   op_cost(0);
 5338   format %{ %}
 5339   interface(REG_INTER);
 5340 %}
 5341 
 5342 operand vRegD_V12()
 5343 %{
 5344   constraint(ALLOC_IN_RC(v12_reg));
 5345   match(RegD);
 5346   op_cost(0);
 5347   format %{ %}
 5348   interface(REG_INTER);
 5349 %}
 5350 
 5351 operand vRegD_V13()
 5352 %{
 5353   constraint(ALLOC_IN_RC(v13_reg));
 5354   match(RegD);
 5355   op_cost(0);
 5356   format %{ %}
 5357   interface(REG_INTER);
 5358 %}
 5359 
 5360 operand vRegD_V14()
 5361 %{
 5362   constraint(ALLOC_IN_RC(v14_reg));
 5363   match(RegD);
 5364   op_cost(0);
 5365   format %{ %}
 5366   interface(REG_INTER);
 5367 %}
 5368 
 5369 operand vRegD_V15()
 5370 %{
 5371   constraint(ALLOC_IN_RC(v15_reg));
 5372   match(RegD);
 5373   op_cost(0);
 5374   format %{ %}
 5375   interface(REG_INTER);
 5376 %}
 5377 
 5378 operand vRegD_V16()
 5379 %{
 5380   constraint(ALLOC_IN_RC(v16_reg));
 5381   match(RegD);
 5382   op_cost(0);
 5383   format %{ %}
 5384   interface(REG_INTER);
 5385 %}
 5386 
 5387 operand vRegD_V17()
 5388 %{
 5389   constraint(ALLOC_IN_RC(v17_reg));
 5390   match(RegD);
 5391   op_cost(0);
 5392   format %{ %}
 5393   interface(REG_INTER);
 5394 %}
 5395 
 5396 operand vRegD_V18()
 5397 %{
 5398   constraint(ALLOC_IN_RC(v18_reg));
 5399   match(RegD);
 5400   op_cost(0);
 5401   format %{ %}
 5402   interface(REG_INTER);
 5403 %}
 5404 
 5405 operand vRegD_V19()
 5406 %{
 5407   constraint(ALLOC_IN_RC(v19_reg));
 5408   match(RegD);
 5409   op_cost(0);
 5410   format %{ %}
 5411   interface(REG_INTER);
 5412 %}
 5413 
 5414 operand vRegD_V20()
 5415 %{
 5416   constraint(ALLOC_IN_RC(v20_reg));
 5417   match(RegD);
 5418   op_cost(0);
 5419   format %{ %}
 5420   interface(REG_INTER);
 5421 %}
 5422 
 5423 operand vRegD_V21()
 5424 %{
 5425   constraint(ALLOC_IN_RC(v21_reg));
 5426   match(RegD);
 5427   op_cost(0);
 5428   format %{ %}
 5429   interface(REG_INTER);
 5430 %}
 5431 
 5432 operand vRegD_V22()
 5433 %{
 5434   constraint(ALLOC_IN_RC(v22_reg));
 5435   match(RegD);
 5436   op_cost(0);
 5437   format %{ %}
 5438   interface(REG_INTER);
 5439 %}
 5440 
 5441 operand vRegD_V23()
 5442 %{
 5443   constraint(ALLOC_IN_RC(v23_reg));
 5444   match(RegD);
 5445   op_cost(0);
 5446   format %{ %}
 5447   interface(REG_INTER);
 5448 %}
 5449 
 5450 operand vRegD_V24()
 5451 %{
 5452   constraint(ALLOC_IN_RC(v24_reg));
 5453   match(RegD);
 5454   op_cost(0);
 5455   format %{ %}
 5456   interface(REG_INTER);
 5457 %}
 5458 
 5459 operand vRegD_V25()
 5460 %{
 5461   constraint(ALLOC_IN_RC(v25_reg));
 5462   match(RegD);
 5463   op_cost(0);
 5464   format %{ %}
 5465   interface(REG_INTER);
 5466 %}
 5467 
 5468 operand vRegD_V26()
 5469 %{
 5470   constraint(ALLOC_IN_RC(v26_reg));
 5471   match(RegD);
 5472   op_cost(0);
 5473   format %{ %}
 5474   interface(REG_INTER);
 5475 %}
 5476 
 5477 operand vRegD_V27()
 5478 %{
 5479   constraint(ALLOC_IN_RC(v27_reg));
 5480   match(RegD);
 5481   op_cost(0);
 5482   format %{ %}
 5483   interface(REG_INTER);
 5484 %}
 5485 
 5486 operand vRegD_V28()
 5487 %{
 5488   constraint(ALLOC_IN_RC(v28_reg));
 5489   match(RegD);
 5490   op_cost(0);
 5491   format %{ %}
 5492   interface(REG_INTER);
 5493 %}
 5494 
 5495 operand vRegD_V29()
 5496 %{
 5497   constraint(ALLOC_IN_RC(v29_reg));
 5498   match(RegD);
 5499   op_cost(0);
 5500   format %{ %}
 5501   interface(REG_INTER);
 5502 %}
 5503 
 5504 operand vRegD_V30()
 5505 %{
 5506   constraint(ALLOC_IN_RC(v30_reg));
 5507   match(RegD);
 5508   op_cost(0);
 5509   format %{ %}
 5510   interface(REG_INTER);
 5511 %}
 5512 
 5513 operand vRegD_V31()
 5514 %{
 5515   constraint(ALLOC_IN_RC(v31_reg));
 5516   match(RegD);
 5517   op_cost(0);
 5518   format %{ %}
 5519   interface(REG_INTER);
 5520 %}
 5521 
 5522 operand pReg()
 5523 %{
 5524   constraint(ALLOC_IN_RC(pr_reg));
 5525   match(RegVectMask);
 5526   op_cost(0);
 5527   format %{ %}
 5528   interface(REG_INTER);
 5529 %}
 5530 
 5531 operand pRegGov()
 5532 %{
 5533   constraint(ALLOC_IN_RC(gov_pr));
 5534   match(RegVectMask);
 5535   op_cost(0);
 5536   format %{ %}
 5537   interface(REG_INTER);
 5538 %}
 5539 
 5540 // Flags register, used as output of signed compare instructions
 5541 
 5542 // note that on AArch64 we also use this register as the output for
 5543 // for floating point compare instructions (CmpF CmpD). this ensures
 5544 // that ordered inequality tests use GT, GE, LT or LE none of which
 5545 // pass through cases where the result is unordered i.e. one or both
 5546 // inputs to the compare is a NaN. this means that the ideal code can
 5547 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5548 // (where the comparison should always fail). EQ and NE tests are
 5549 // always generated in ideal code so that unordered folds into the NE
 5550 // case, matching the behaviour of AArch64 NE.
 5551 //
 5552 // This differs from x86 where the outputs of FP compares use a
 5553 // special FP flags registers and where compares based on this
 5554 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5555 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5556 // to explicitly handle the unordered case in branches. x86 also has
 5557 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5558 
 5559 operand rFlagsReg()
 5560 %{
 5561   constraint(ALLOC_IN_RC(int_flags));
 5562   match(RegFlags);
 5563 
 5564   op_cost(0);
 5565   format %{ "RFLAGS" %}
 5566   interface(REG_INTER);
 5567 %}
 5568 
 5569 // Flags register, used as output of unsigned compare instructions
 5570 operand rFlagsRegU()
 5571 %{
 5572   constraint(ALLOC_IN_RC(int_flags));
 5573   match(RegFlags);
 5574 
 5575   op_cost(0);
 5576   format %{ "RFLAGSU" %}
 5577   interface(REG_INTER);
 5578 %}
 5579 
 5580 // Special Registers
 5581 
 5582 // Method Register
 5583 operand inline_cache_RegP(iRegP reg)
 5584 %{
 5585   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5586   match(reg);
 5587   match(iRegPNoSp);
 5588   op_cost(0);
 5589   format %{ %}
 5590   interface(REG_INTER);
 5591 %}
 5592 
 5593 // Thread Register
 5594 operand thread_RegP(iRegP reg)
 5595 %{
 5596   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5597   match(reg);
 5598   op_cost(0);
 5599   format %{ %}
 5600   interface(REG_INTER);
 5601 %}
 5602 
 5603 operand lr_RegP(iRegP reg)
 5604 %{
 5605   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5606   match(reg);
 5607   op_cost(0);
 5608   format %{ %}
 5609   interface(REG_INTER);
 5610 %}
 5611 
 5612 //----------Memory Operands----------------------------------------------------
 5613 
 5614 operand indirect(iRegP reg)
 5615 %{
 5616   constraint(ALLOC_IN_RC(ptr_reg));
 5617   match(reg);
 5618   op_cost(0);
 5619   format %{ "[$reg]" %}
 5620   interface(MEMORY_INTER) %{
 5621     base($reg);
 5622     index(0xffffffff);
 5623     scale(0x0);
 5624     disp(0x0);
 5625   %}
 5626 %}
 5627 
 5628 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5629 %{
 5630   constraint(ALLOC_IN_RC(ptr_reg));
 5631   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5632   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5633   op_cost(0);
 5634   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5635   interface(MEMORY_INTER) %{
 5636     base($reg);
 5637     index($ireg);
 5638     scale($scale);
 5639     disp(0x0);
 5640   %}
 5641 %}
 5642 
 5643 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5644 %{
 5645   constraint(ALLOC_IN_RC(ptr_reg));
 5646   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5647   match(AddP reg (LShiftL lreg scale));
 5648   op_cost(0);
 5649   format %{ "$reg, $lreg lsl($scale)" %}
 5650   interface(MEMORY_INTER) %{
 5651     base($reg);
 5652     index($lreg);
 5653     scale($scale);
 5654     disp(0x0);
 5655   %}
 5656 %}
 5657 
 5658 operand indIndexI2L(iRegP reg, iRegI ireg)
 5659 %{
 5660   constraint(ALLOC_IN_RC(ptr_reg));
 5661   match(AddP reg (ConvI2L ireg));
 5662   op_cost(0);
 5663   format %{ "$reg, $ireg, 0, I2L" %}
 5664   interface(MEMORY_INTER) %{
 5665     base($reg);
 5666     index($ireg);
 5667     scale(0x0);
 5668     disp(0x0);
 5669   %}
 5670 %}
 5671 
 5672 operand indIndex(iRegP reg, iRegL lreg)
 5673 %{
 5674   constraint(ALLOC_IN_RC(ptr_reg));
 5675   match(AddP reg lreg);
 5676   op_cost(0);
 5677   format %{ "$reg, $lreg" %}
 5678   interface(MEMORY_INTER) %{
 5679     base($reg);
 5680     index($lreg);
 5681     scale(0x0);
 5682     disp(0x0);
 5683   %}
 5684 %}
 5685 
 5686 operand indOffI(iRegP reg, immIOffset off)
 5687 %{
 5688   constraint(ALLOC_IN_RC(ptr_reg));
 5689   match(AddP reg off);
 5690   op_cost(0);
 5691   format %{ "[$reg, $off]" %}
 5692   interface(MEMORY_INTER) %{
 5693     base($reg);
 5694     index(0xffffffff);
 5695     scale(0x0);
 5696     disp($off);
 5697   %}
 5698 %}
 5699 
 5700 operand indOffI1(iRegP reg, immIOffset1 off)
 5701 %{
 5702   constraint(ALLOC_IN_RC(ptr_reg));
 5703   match(AddP reg off);
 5704   op_cost(0);
 5705   format %{ "[$reg, $off]" %}
 5706   interface(MEMORY_INTER) %{
 5707     base($reg);
 5708     index(0xffffffff);
 5709     scale(0x0);
 5710     disp($off);
 5711   %}
 5712 %}
 5713 
 5714 operand indOffI2(iRegP reg, immIOffset2 off)
 5715 %{
 5716   constraint(ALLOC_IN_RC(ptr_reg));
 5717   match(AddP reg off);
 5718   op_cost(0);
 5719   format %{ "[$reg, $off]" %}
 5720   interface(MEMORY_INTER) %{
 5721     base($reg);
 5722     index(0xffffffff);
 5723     scale(0x0);
 5724     disp($off);
 5725   %}
 5726 %}
 5727 
 5728 operand indOffI4(iRegP reg, immIOffset4 off)
 5729 %{
 5730   constraint(ALLOC_IN_RC(ptr_reg));
 5731   match(AddP reg off);
 5732   op_cost(0);
 5733   format %{ "[$reg, $off]" %}
 5734   interface(MEMORY_INTER) %{
 5735     base($reg);
 5736     index(0xffffffff);
 5737     scale(0x0);
 5738     disp($off);
 5739   %}
 5740 %}
 5741 
 5742 operand indOffI8(iRegP reg, immIOffset8 off)
 5743 %{
 5744   constraint(ALLOC_IN_RC(ptr_reg));
 5745   match(AddP reg off);
 5746   op_cost(0);
 5747   format %{ "[$reg, $off]" %}
 5748   interface(MEMORY_INTER) %{
 5749     base($reg);
 5750     index(0xffffffff);
 5751     scale(0x0);
 5752     disp($off);
 5753   %}
 5754 %}
 5755 
 5756 operand indOffI16(iRegP reg, immIOffset16 off)
 5757 %{
 5758   constraint(ALLOC_IN_RC(ptr_reg));
 5759   match(AddP reg off);
 5760   op_cost(0);
 5761   format %{ "[$reg, $off]" %}
 5762   interface(MEMORY_INTER) %{
 5763     base($reg);
 5764     index(0xffffffff);
 5765     scale(0x0);
 5766     disp($off);
 5767   %}
 5768 %}
 5769 
 5770 operand indOffL(iRegP reg, immLoffset off)
 5771 %{
 5772   constraint(ALLOC_IN_RC(ptr_reg));
 5773   match(AddP reg off);
 5774   op_cost(0);
 5775   format %{ "[$reg, $off]" %}
 5776   interface(MEMORY_INTER) %{
 5777     base($reg);
 5778     index(0xffffffff);
 5779     scale(0x0);
 5780     disp($off);
 5781   %}
 5782 %}
 5783 
 5784 operand indOffL1(iRegP reg, immLoffset1 off)
 5785 %{
 5786   constraint(ALLOC_IN_RC(ptr_reg));
 5787   match(AddP reg off);
 5788   op_cost(0);
 5789   format %{ "[$reg, $off]" %}
 5790   interface(MEMORY_INTER) %{
 5791     base($reg);
 5792     index(0xffffffff);
 5793     scale(0x0);
 5794     disp($off);
 5795   %}
 5796 %}
 5797 
 5798 operand indOffL2(iRegP reg, immLoffset2 off)
 5799 %{
 5800   constraint(ALLOC_IN_RC(ptr_reg));
 5801   match(AddP reg off);
 5802   op_cost(0);
 5803   format %{ "[$reg, $off]" %}
 5804   interface(MEMORY_INTER) %{
 5805     base($reg);
 5806     index(0xffffffff);
 5807     scale(0x0);
 5808     disp($off);
 5809   %}
 5810 %}
 5811 
 5812 operand indOffL4(iRegP reg, immLoffset4 off)
 5813 %{
 5814   constraint(ALLOC_IN_RC(ptr_reg));
 5815   match(AddP reg off);
 5816   op_cost(0);
 5817   format %{ "[$reg, $off]" %}
 5818   interface(MEMORY_INTER) %{
 5819     base($reg);
 5820     index(0xffffffff);
 5821     scale(0x0);
 5822     disp($off);
 5823   %}
 5824 %}
 5825 
 5826 operand indOffL8(iRegP reg, immLoffset8 off)
 5827 %{
 5828   constraint(ALLOC_IN_RC(ptr_reg));
 5829   match(AddP reg off);
 5830   op_cost(0);
 5831   format %{ "[$reg, $off]" %}
 5832   interface(MEMORY_INTER) %{
 5833     base($reg);
 5834     index(0xffffffff);
 5835     scale(0x0);
 5836     disp($off);
 5837   %}
 5838 %}
 5839 
 5840 operand indOffL16(iRegP reg, immLoffset16 off)
 5841 %{
 5842   constraint(ALLOC_IN_RC(ptr_reg));
 5843   match(AddP reg off);
 5844   op_cost(0);
 5845   format %{ "[$reg, $off]" %}
 5846   interface(MEMORY_INTER) %{
 5847     base($reg);
 5848     index(0xffffffff);
 5849     scale(0x0);
 5850     disp($off);
 5851   %}
 5852 %}
 5853 
 5854 operand indirectN(iRegN reg)
 5855 %{
 5856   predicate(CompressedOops::shift() == 0);
 5857   constraint(ALLOC_IN_RC(ptr_reg));
 5858   match(DecodeN reg);
 5859   op_cost(0);
 5860   format %{ "[$reg]\t# narrow" %}
 5861   interface(MEMORY_INTER) %{
 5862     base($reg);
 5863     index(0xffffffff);
 5864     scale(0x0);
 5865     disp(0x0);
 5866   %}
 5867 %}
 5868 
 5869 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5870 %{
 5871   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5872   constraint(ALLOC_IN_RC(ptr_reg));
 5873   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5874   op_cost(0);
 5875   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5876   interface(MEMORY_INTER) %{
 5877     base($reg);
 5878     index($ireg);
 5879     scale($scale);
 5880     disp(0x0);
 5881   %}
 5882 %}
 5883 
 5884 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5885 %{
 5886   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5887   constraint(ALLOC_IN_RC(ptr_reg));
 5888   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5889   op_cost(0);
 5890   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5891   interface(MEMORY_INTER) %{
 5892     base($reg);
 5893     index($lreg);
 5894     scale($scale);
 5895     disp(0x0);
 5896   %}
 5897 %}
 5898 
 5899 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5900 %{
 5901   predicate(CompressedOops::shift() == 0);
 5902   constraint(ALLOC_IN_RC(ptr_reg));
 5903   match(AddP (DecodeN reg) (ConvI2L ireg));
 5904   op_cost(0);
 5905   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5906   interface(MEMORY_INTER) %{
 5907     base($reg);
 5908     index($ireg);
 5909     scale(0x0);
 5910     disp(0x0);
 5911   %}
 5912 %}
 5913 
 5914 operand indIndexN(iRegN reg, iRegL lreg)
 5915 %{
 5916   predicate(CompressedOops::shift() == 0);
 5917   constraint(ALLOC_IN_RC(ptr_reg));
 5918   match(AddP (DecodeN reg) lreg);
 5919   op_cost(0);
 5920   format %{ "$reg, $lreg\t# narrow" %}
 5921   interface(MEMORY_INTER) %{
 5922     base($reg);
 5923     index($lreg);
 5924     scale(0x0);
 5925     disp(0x0);
 5926   %}
 5927 %}
 5928 
 5929 operand indOffIN(iRegN reg, immIOffset off)
 5930 %{
 5931   predicate(CompressedOops::shift() == 0);
 5932   constraint(ALLOC_IN_RC(ptr_reg));
 5933   match(AddP (DecodeN reg) off);
 5934   op_cost(0);
 5935   format %{ "[$reg, $off]\t# narrow" %}
 5936   interface(MEMORY_INTER) %{
 5937     base($reg);
 5938     index(0xffffffff);
 5939     scale(0x0);
 5940     disp($off);
 5941   %}
 5942 %}
 5943 
 5944 operand indOffLN(iRegN reg, immLoffset off)
 5945 %{
 5946   predicate(CompressedOops::shift() == 0);
 5947   constraint(ALLOC_IN_RC(ptr_reg));
 5948   match(AddP (DecodeN reg) off);
 5949   op_cost(0);
 5950   format %{ "[$reg, $off]\t# narrow" %}
 5951   interface(MEMORY_INTER) %{
 5952     base($reg);
 5953     index(0xffffffff);
 5954     scale(0x0);
 5955     disp($off);
 5956   %}
 5957 %}
 5958 
 5959 
 5960 
 5961 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 5962 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 5963 %{
 5964   constraint(ALLOC_IN_RC(ptr_reg));
 5965   match(AddP reg off);
 5966   op_cost(0);
 5967   format %{ "[$reg, $off]" %}
 5968   interface(MEMORY_INTER) %{
 5969     base($reg);
 5970     index(0xffffffff);
 5971     scale(0x0);
 5972     disp($off);
 5973   %}
 5974 %}
 5975 
 5976 //----------Special Memory Operands--------------------------------------------
 5977 // Stack Slot Operand - This operand is used for loading and storing temporary
 5978 //                      values on the stack where a match requires a value to
 5979 //                      flow through memory.
 5980 operand stackSlotP(sRegP reg)
 5981 %{
 5982   constraint(ALLOC_IN_RC(stack_slots));
 5983   op_cost(100);
 5984   // No match rule because this operand is only generated in matching
 5985   // match(RegP);
 5986   format %{ "[$reg]" %}
 5987   interface(MEMORY_INTER) %{
 5988     base(0x1e);  // RSP
 5989     index(0x0);  // No Index
 5990     scale(0x0);  // No Scale
 5991     disp($reg);  // Stack Offset
 5992   %}
 5993 %}
 5994 
 5995 operand stackSlotI(sRegI reg)
 5996 %{
 5997   constraint(ALLOC_IN_RC(stack_slots));
 5998   // No match rule because this operand is only generated in matching
 5999   // match(RegI);
 6000   format %{ "[$reg]" %}
 6001   interface(MEMORY_INTER) %{
 6002     base(0x1e);  // RSP
 6003     index(0x0);  // No Index
 6004     scale(0x0);  // No Scale
 6005     disp($reg);  // Stack Offset
 6006   %}
 6007 %}
 6008 
 6009 operand stackSlotF(sRegF reg)
 6010 %{
 6011   constraint(ALLOC_IN_RC(stack_slots));
 6012   // No match rule because this operand is only generated in matching
 6013   // match(RegF);
 6014   format %{ "[$reg]" %}
 6015   interface(MEMORY_INTER) %{
 6016     base(0x1e);  // RSP
 6017     index(0x0);  // No Index
 6018     scale(0x0);  // No Scale
 6019     disp($reg);  // Stack Offset
 6020   %}
 6021 %}
 6022 
 6023 operand stackSlotD(sRegD reg)
 6024 %{
 6025   constraint(ALLOC_IN_RC(stack_slots));
 6026   // No match rule because this operand is only generated in matching
 6027   // match(RegD);
 6028   format %{ "[$reg]" %}
 6029   interface(MEMORY_INTER) %{
 6030     base(0x1e);  // RSP
 6031     index(0x0);  // No Index
 6032     scale(0x0);  // No Scale
 6033     disp($reg);  // Stack Offset
 6034   %}
 6035 %}
 6036 
 6037 operand stackSlotL(sRegL reg)
 6038 %{
 6039   constraint(ALLOC_IN_RC(stack_slots));
 6040   // No match rule because this operand is only generated in matching
 6041   // match(RegL);
 6042   format %{ "[$reg]" %}
 6043   interface(MEMORY_INTER) %{
 6044     base(0x1e);  // RSP
 6045     index(0x0);  // No Index
 6046     scale(0x0);  // No Scale
 6047     disp($reg);  // Stack Offset
 6048   %}
 6049 %}
 6050 
 6051 // Operands for expressing Control Flow
 6052 // NOTE: Label is a predefined operand which should not be redefined in
 6053 //       the AD file. It is generically handled within the ADLC.
 6054 
 6055 //----------Conditional Branch Operands----------------------------------------
 6056 // Comparison Op  - This is the operation of the comparison, and is limited to
 6057 //                  the following set of codes:
 6058 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6059 //
 6060 // Other attributes of the comparison, such as unsignedness, are specified
 6061 // by the comparison instruction that sets a condition code flags register.
 6062 // That result is represented by a flags operand whose subtype is appropriate
 6063 // to the unsignedness (etc.) of the comparison.
 6064 //
 6065 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6066 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6067 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6068 
 6069 // used for signed integral comparisons and fp comparisons
 6070 
 6071 operand cmpOp()
 6072 %{
 6073   match(Bool);
 6074 
 6075   format %{ "" %}
 6076   interface(COND_INTER) %{
 6077     equal(0x0, "eq");
 6078     not_equal(0x1, "ne");
 6079     less(0xb, "lt");
 6080     greater_equal(0xa, "ge");
 6081     less_equal(0xd, "le");
 6082     greater(0xc, "gt");
 6083     overflow(0x6, "vs");
 6084     no_overflow(0x7, "vc");
 6085   %}
 6086 %}
 6087 
 6088 // used for unsigned integral comparisons
 6089 
 6090 operand cmpOpU()
 6091 %{
 6092   match(Bool);
 6093 
 6094   format %{ "" %}
 6095   interface(COND_INTER) %{
 6096     equal(0x0, "eq");
 6097     not_equal(0x1, "ne");
 6098     less(0x3, "lo");
 6099     greater_equal(0x2, "hs");
 6100     less_equal(0x9, "ls");
 6101     greater(0x8, "hi");
 6102     overflow(0x6, "vs");
 6103     no_overflow(0x7, "vc");
 6104   %}
 6105 %}
 6106 
 6107 // used for certain integral comparisons which can be
 6108 // converted to cbxx or tbxx instructions
 6109 
 6110 operand cmpOpEqNe()
 6111 %{
 6112   match(Bool);
 6113   op_cost(0);
 6114   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6115             || n->as_Bool()->_test._test == BoolTest::eq);
 6116 
 6117   format %{ "" %}
 6118   interface(COND_INTER) %{
 6119     equal(0x0, "eq");
 6120     not_equal(0x1, "ne");
 6121     less(0xb, "lt");
 6122     greater_equal(0xa, "ge");
 6123     less_equal(0xd, "le");
 6124     greater(0xc, "gt");
 6125     overflow(0x6, "vs");
 6126     no_overflow(0x7, "vc");
 6127   %}
 6128 %}
 6129 
 6130 // used for certain integral comparisons which can be
 6131 // converted to cbxx or tbxx instructions
 6132 
 6133 operand cmpOpLtGe()
 6134 %{
 6135   match(Bool);
 6136   op_cost(0);
 6137 
 6138   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6139             || n->as_Bool()->_test._test == BoolTest::ge);
 6140 
 6141   format %{ "" %}
 6142   interface(COND_INTER) %{
 6143     equal(0x0, "eq");
 6144     not_equal(0x1, "ne");
 6145     less(0xb, "lt");
 6146     greater_equal(0xa, "ge");
 6147     less_equal(0xd, "le");
 6148     greater(0xc, "gt");
 6149     overflow(0x6, "vs");
 6150     no_overflow(0x7, "vc");
 6151   %}
 6152 %}
 6153 
 6154 // used for certain unsigned integral comparisons which can be
 6155 // converted to cbxx or tbxx instructions
 6156 
 6157 operand cmpOpUEqNeLtGe()
 6158 %{
 6159   match(Bool);
 6160   op_cost(0);
 6161 
 6162   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6163             || n->as_Bool()->_test._test == BoolTest::ne
 6164             || n->as_Bool()->_test._test == BoolTest::lt
 6165             || n->as_Bool()->_test._test == BoolTest::ge);
 6166 
 6167   format %{ "" %}
 6168   interface(COND_INTER) %{
 6169     equal(0x0, "eq");
 6170     not_equal(0x1, "ne");
 6171     less(0xb, "lt");
 6172     greater_equal(0xa, "ge");
 6173     less_equal(0xd, "le");
 6174     greater(0xc, "gt");
 6175     overflow(0x6, "vs");
 6176     no_overflow(0x7, "vc");
 6177   %}
 6178 %}
 6179 
 6180 // Special operand allowing long args to int ops to be truncated for free
 6181 
 6182 operand iRegL2I(iRegL reg) %{
 6183 
 6184   op_cost(0);
 6185 
 6186   match(ConvL2I reg);
 6187 
 6188   format %{ "l2i($reg)" %}
 6189 
 6190   interface(REG_INTER)
 6191 %}
 6192 
 6193 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6194 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6195 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6196 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6197 
 6198 //----------OPERAND CLASSES----------------------------------------------------
 6199 // Operand Classes are groups of operands that are used as to simplify
 6200 // instruction definitions by not requiring the AD writer to specify
 6201 // separate instructions for every form of operand when the
 6202 // instruction accepts multiple operand types with the same basic
 6203 // encoding and format. The classic case of this is memory operands.
 6204 
 6205 // memory is used to define read/write location for load/store
 6206 // instruction defs. we can turn a memory op into an Address
 6207 
 6208 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6209                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6210 
 6211 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6212                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6213 
 6214 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6215                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6216 
 6217 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6218                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6219 
 6220 // All of the memory operands. For the pipeline description.
 6221 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6222                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6223                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6224 
 6225 
 6226 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6227 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6228 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6229 // can be elided because the 32-bit instruction will just employ the
 6230 // lower 32 bits anyway.
 6231 //
 6232 // n.b. this does not elide all L2I conversions. if the truncated
 6233 // value is consumed by more than one operation then the ConvL2I
 6234 // cannot be bundled into the consuming nodes so an l2i gets planted
 6235 // (actually a movw $dst $src) and the downstream instructions consume
 6236 // the result of the l2i as an iRegI input. That's a shame since the
 6237 // movw is actually redundant but its not too costly.
 6238 
 6239 opclass iRegIorL2I(iRegI, iRegL2I);
 6240 
 6241 //----------PIPELINE-----------------------------------------------------------
 6242 // Rules which define the behavior of the target architectures pipeline.
 6243 
 6244 // For specific pipelines, eg A53, define the stages of that pipeline
 6245 //pipe_desc(ISS, EX1, EX2, WR);
 6246 #define ISS S0
 6247 #define EX1 S1
 6248 #define EX2 S2
 6249 #define WR  S3
 6250 
 6251 // Integer ALU reg operation
 6252 pipeline %{
 6253 
 6254 attributes %{
 6255   // ARM instructions are of fixed length
 6256   fixed_size_instructions;        // Fixed size instructions TODO does
 6257   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6258   // ARM instructions come in 32-bit word units
 6259   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6260   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6261   instruction_fetch_units = 1;       // of 64 bytes
 6262 
 6263   // List of nop instructions
 6264   nops( MachNop );
 6265 %}
 6266 
 6267 // We don't use an actual pipeline model so don't care about resources
 6268 // or description. we do use pipeline classes to introduce fixed
 6269 // latencies
 6270 
 6271 //----------RESOURCES----------------------------------------------------------
 6272 // Resources are the functional units available to the machine
 6273 
 6274 resources( INS0, INS1, INS01 = INS0 | INS1,
 6275            ALU0, ALU1, ALU = ALU0 | ALU1,
 6276            MAC,
 6277            DIV,
 6278            BRANCH,
 6279            LDST,
 6280            NEON_FP);
 6281 
 6282 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6283 // Pipeline Description specifies the stages in the machine's pipeline
 6284 
 6285 // Define the pipeline as a generic 6 stage pipeline
 6286 pipe_desc(S0, S1, S2, S3, S4, S5);
 6287 
 6288 //----------PIPELINE CLASSES---------------------------------------------------
 6289 // Pipeline Classes describe the stages in which input and output are
 6290 // referenced by the hardware pipeline.
 6291 
 6292 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6293 %{
 6294   single_instruction;
 6295   src1   : S1(read);
 6296   src2   : S2(read);
 6297   dst    : S5(write);
 6298   INS01  : ISS;
 6299   NEON_FP : S5;
 6300 %}
 6301 
 6302 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6303 %{
 6304   single_instruction;
 6305   src1   : S1(read);
 6306   src2   : S2(read);
 6307   dst    : S5(write);
 6308   INS01  : ISS;
 6309   NEON_FP : S5;
 6310 %}
 6311 
 6312 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6313 %{
 6314   single_instruction;
 6315   src    : S1(read);
 6316   dst    : S5(write);
 6317   INS01  : ISS;
 6318   NEON_FP : S5;
 6319 %}
 6320 
 6321 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6322 %{
 6323   single_instruction;
 6324   src    : S1(read);
 6325   dst    : S5(write);
 6326   INS01  : ISS;
 6327   NEON_FP : S5;
 6328 %}
 6329 
 6330 pipe_class fp_d2f(vRegF dst, vRegD src)
 6331 %{
 6332   single_instruction;
 6333   src    : S1(read);
 6334   dst    : S5(write);
 6335   INS01  : ISS;
 6336   NEON_FP : S5;
 6337 %}
 6338 
 6339 pipe_class fp_f2d(vRegD dst, vRegF src)
 6340 %{
 6341   single_instruction;
 6342   src    : S1(read);
 6343   dst    : S5(write);
 6344   INS01  : ISS;
 6345   NEON_FP : S5;
 6346 %}
 6347 
 6348 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6349 %{
 6350   single_instruction;
 6351   src    : S1(read);
 6352   dst    : S5(write);
 6353   INS01  : ISS;
 6354   NEON_FP : S5;
 6355 %}
 6356 
 6357 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6358 %{
 6359   single_instruction;
 6360   src    : S1(read);
 6361   dst    : S5(write);
 6362   INS01  : ISS;
 6363   NEON_FP : S5;
 6364 %}
 6365 
 6366 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6367 %{
 6368   single_instruction;
 6369   src    : S1(read);
 6370   dst    : S5(write);
 6371   INS01  : ISS;
 6372   NEON_FP : S5;
 6373 %}
 6374 
 6375 pipe_class fp_l2f(vRegF dst, iRegL src)
 6376 %{
 6377   single_instruction;
 6378   src    : S1(read);
 6379   dst    : S5(write);
 6380   INS01  : ISS;
 6381   NEON_FP : S5;
 6382 %}
 6383 
 6384 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6385 %{
 6386   single_instruction;
 6387   src    : S1(read);
 6388   dst    : S5(write);
 6389   INS01  : ISS;
 6390   NEON_FP : S5;
 6391 %}
 6392 
 6393 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6394 %{
 6395   single_instruction;
 6396   src    : S1(read);
 6397   dst    : S5(write);
 6398   INS01  : ISS;
 6399   NEON_FP : S5;
 6400 %}
 6401 
 6402 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6403 %{
 6404   single_instruction;
 6405   src    : S1(read);
 6406   dst    : S5(write);
 6407   INS01  : ISS;
 6408   NEON_FP : S5;
 6409 %}
 6410 
 6411 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6412 %{
 6413   single_instruction;
 6414   src    : S1(read);
 6415   dst    : S5(write);
 6416   INS01  : ISS;
 6417   NEON_FP : S5;
 6418 %}
 6419 
 6420 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6421 %{
 6422   single_instruction;
 6423   src1   : S1(read);
 6424   src2   : S2(read);
 6425   dst    : S5(write);
 6426   INS0   : ISS;
 6427   NEON_FP : S5;
 6428 %}
 6429 
 6430 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6431 %{
 6432   single_instruction;
 6433   src1   : S1(read);
 6434   src2   : S2(read);
 6435   dst    : S5(write);
 6436   INS0   : ISS;
 6437   NEON_FP : S5;
 6438 %}
 6439 
 6440 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6441 %{
 6442   single_instruction;
 6443   cr     : S1(read);
 6444   src1   : S1(read);
 6445   src2   : S1(read);
 6446   dst    : S3(write);
 6447   INS01  : ISS;
 6448   NEON_FP : S3;
 6449 %}
 6450 
 6451 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6452 %{
 6453   single_instruction;
 6454   cr     : S1(read);
 6455   src1   : S1(read);
 6456   src2   : S1(read);
 6457   dst    : S3(write);
 6458   INS01  : ISS;
 6459   NEON_FP : S3;
 6460 %}
 6461 
 6462 pipe_class fp_imm_s(vRegF dst)
 6463 %{
 6464   single_instruction;
 6465   dst    : S3(write);
 6466   INS01  : ISS;
 6467   NEON_FP : S3;
 6468 %}
 6469 
 6470 pipe_class fp_imm_d(vRegD dst)
 6471 %{
 6472   single_instruction;
 6473   dst    : S3(write);
 6474   INS01  : ISS;
 6475   NEON_FP : S3;
 6476 %}
 6477 
 6478 pipe_class fp_load_constant_s(vRegF dst)
 6479 %{
 6480   single_instruction;
 6481   dst    : S4(write);
 6482   INS01  : ISS;
 6483   NEON_FP : S4;
 6484 %}
 6485 
 6486 pipe_class fp_load_constant_d(vRegD dst)
 6487 %{
 6488   single_instruction;
 6489   dst    : S4(write);
 6490   INS01  : ISS;
 6491   NEON_FP : S4;
 6492 %}
 6493 
 6494 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6495 %{
 6496   single_instruction;
 6497   dst    : S5(write);
 6498   src1   : S1(read);
 6499   src2   : S1(read);
 6500   INS01  : ISS;
 6501   NEON_FP : S5;
 6502 %}
 6503 
 6504 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6505 %{
 6506   single_instruction;
 6507   dst    : S5(write);
 6508   src1   : S1(read);
 6509   src2   : S1(read);
 6510   INS0   : ISS;
 6511   NEON_FP : S5;
 6512 %}
 6513 
 6514 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6515 %{
 6516   single_instruction;
 6517   dst    : S5(write);
 6518   src1   : S1(read);
 6519   src2   : S1(read);
 6520   dst    : S1(read);
 6521   INS01  : ISS;
 6522   NEON_FP : S5;
 6523 %}
 6524 
 6525 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6526 %{
 6527   single_instruction;
 6528   dst    : S5(write);
 6529   src1   : S1(read);
 6530   src2   : S1(read);
 6531   dst    : S1(read);
 6532   INS0   : ISS;
 6533   NEON_FP : S5;
 6534 %}
 6535 
 6536 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6537 %{
 6538   single_instruction;
 6539   dst    : S4(write);
 6540   src1   : S2(read);
 6541   src2   : S2(read);
 6542   INS01  : ISS;
 6543   NEON_FP : S4;
 6544 %}
 6545 
 6546 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6547 %{
 6548   single_instruction;
 6549   dst    : S4(write);
 6550   src1   : S2(read);
 6551   src2   : S2(read);
 6552   INS0   : ISS;
 6553   NEON_FP : S4;
 6554 %}
 6555 
 6556 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6557 %{
 6558   single_instruction;
 6559   dst    : S3(write);
 6560   src1   : S2(read);
 6561   src2   : S2(read);
 6562   INS01  : ISS;
 6563   NEON_FP : S3;
 6564 %}
 6565 
 6566 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6567 %{
 6568   single_instruction;
 6569   dst    : S3(write);
 6570   src1   : S2(read);
 6571   src2   : S2(read);
 6572   INS0   : ISS;
 6573   NEON_FP : S3;
 6574 %}
 6575 
 6576 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6577 %{
 6578   single_instruction;
 6579   dst    : S3(write);
 6580   src    : S1(read);
 6581   shift  : S1(read);
 6582   INS01  : ISS;
 6583   NEON_FP : S3;
 6584 %}
 6585 
 6586 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6587 %{
 6588   single_instruction;
 6589   dst    : S3(write);
 6590   src    : S1(read);
 6591   shift  : S1(read);
 6592   INS0   : ISS;
 6593   NEON_FP : S3;
 6594 %}
 6595 
 6596 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6597 %{
 6598   single_instruction;
 6599   dst    : S3(write);
 6600   src    : S1(read);
 6601   INS01  : ISS;
 6602   NEON_FP : S3;
 6603 %}
 6604 
 6605 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6606 %{
 6607   single_instruction;
 6608   dst    : S3(write);
 6609   src    : S1(read);
 6610   INS0   : ISS;
 6611   NEON_FP : S3;
 6612 %}
 6613 
 6614 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6615 %{
 6616   single_instruction;
 6617   dst    : S5(write);
 6618   src1   : S1(read);
 6619   src2   : S1(read);
 6620   INS01  : ISS;
 6621   NEON_FP : S5;
 6622 %}
 6623 
 6624 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6625 %{
 6626   single_instruction;
 6627   dst    : S5(write);
 6628   src1   : S1(read);
 6629   src2   : S1(read);
 6630   INS0   : ISS;
 6631   NEON_FP : S5;
 6632 %}
 6633 
 6634 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6635 %{
 6636   single_instruction;
 6637   dst    : S5(write);
 6638   src1   : S1(read);
 6639   src2   : S1(read);
 6640   INS0   : ISS;
 6641   NEON_FP : S5;
 6642 %}
 6643 
 6644 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6645 %{
 6646   single_instruction;
 6647   dst    : S5(write);
 6648   src1   : S1(read);
 6649   src2   : S1(read);
 6650   INS0   : ISS;
 6651   NEON_FP : S5;
 6652 %}
 6653 
 6654 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6655 %{
 6656   single_instruction;
 6657   dst    : S5(write);
 6658   src    : S1(read);
 6659   INS0   : ISS;
 6660   NEON_FP : S5;
 6661 %}
 6662 
 6663 pipe_class vunop_fp64(vecD dst, vecD src)
 6664 %{
 6665   single_instruction;
 6666   dst    : S5(write);
 6667   src    : S1(read);
 6668   INS01  : ISS;
 6669   NEON_FP : S5;
 6670 %}
 6671 
 6672 pipe_class vunop_fp128(vecX dst, vecX src)
 6673 %{
 6674   single_instruction;
 6675   dst    : S5(write);
 6676   src    : S1(read);
 6677   INS0   : ISS;
 6678   NEON_FP : S5;
 6679 %}
 6680 
 6681 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6682 %{
 6683   single_instruction;
 6684   dst    : S3(write);
 6685   src    : S1(read);
 6686   INS01  : ISS;
 6687   NEON_FP : S3;
 6688 %}
 6689 
 6690 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6691 %{
 6692   single_instruction;
 6693   dst    : S3(write);
 6694   src    : S1(read);
 6695   INS01  : ISS;
 6696   NEON_FP : S3;
 6697 %}
 6698 
 6699 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6700 %{
 6701   single_instruction;
 6702   dst    : S3(write);
 6703   src    : S1(read);
 6704   INS01  : ISS;
 6705   NEON_FP : S3;
 6706 %}
 6707 
 6708 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6709 %{
 6710   single_instruction;
 6711   dst    : S3(write);
 6712   src    : S1(read);
 6713   INS01  : ISS;
 6714   NEON_FP : S3;
 6715 %}
 6716 
 6717 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6718 %{
 6719   single_instruction;
 6720   dst    : S3(write);
 6721   src    : S1(read);
 6722   INS01  : ISS;
 6723   NEON_FP : S3;
 6724 %}
 6725 
 6726 pipe_class vmovi_reg_imm64(vecD dst)
 6727 %{
 6728   single_instruction;
 6729   dst    : S3(write);
 6730   INS01  : ISS;
 6731   NEON_FP : S3;
 6732 %}
 6733 
 6734 pipe_class vmovi_reg_imm128(vecX dst)
 6735 %{
 6736   single_instruction;
 6737   dst    : S3(write);
 6738   INS0   : ISS;
 6739   NEON_FP : S3;
 6740 %}
 6741 
 6742 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6743 %{
 6744   single_instruction;
 6745   dst    : S5(write);
 6746   mem    : ISS(read);
 6747   INS01  : ISS;
 6748   NEON_FP : S3;
 6749 %}
 6750 
 6751 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6752 %{
 6753   single_instruction;
 6754   dst    : S5(write);
 6755   mem    : ISS(read);
 6756   INS01  : ISS;
 6757   NEON_FP : S3;
 6758 %}
 6759 
 6760 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6761 %{
 6762   single_instruction;
 6763   mem    : ISS(read);
 6764   src    : S2(read);
 6765   INS01  : ISS;
 6766   NEON_FP : S3;
 6767 %}
 6768 
 6769 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6770 %{
 6771   single_instruction;
 6772   mem    : ISS(read);
 6773   src    : S2(read);
 6774   INS01  : ISS;
 6775   NEON_FP : S3;
 6776 %}
 6777 
 6778 //------- Integer ALU operations --------------------------
 6779 
 6780 // Integer ALU reg-reg operation
 6781 // Operands needed in EX1, result generated in EX2
 6782 // Eg.  ADD     x0, x1, x2
 6783 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6784 %{
 6785   single_instruction;
 6786   dst    : EX2(write);
 6787   src1   : EX1(read);
 6788   src2   : EX1(read);
 6789   INS01  : ISS; // Dual issue as instruction 0 or 1
 6790   ALU    : EX2;
 6791 %}
 6792 
 6793 // Integer ALU reg-reg operation with constant shift
 6794 // Shifted register must be available in LATE_ISS instead of EX1
 6795 // Eg.  ADD     x0, x1, x2, LSL #2
 6796 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6797 %{
 6798   single_instruction;
 6799   dst    : EX2(write);
 6800   src1   : EX1(read);
 6801   src2   : ISS(read);
 6802   INS01  : ISS;
 6803   ALU    : EX2;
 6804 %}
 6805 
 6806 // Integer ALU reg operation with constant shift
 6807 // Eg.  LSL     x0, x1, #shift
 6808 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6809 %{
 6810   single_instruction;
 6811   dst    : EX2(write);
 6812   src1   : ISS(read);
 6813   INS01  : ISS;
 6814   ALU    : EX2;
 6815 %}
 6816 
 6817 // Integer ALU reg-reg operation with variable shift
 6818 // Both operands must be available in LATE_ISS instead of EX1
 6819 // Result is available in EX1 instead of EX2
 6820 // Eg.  LSLV    x0, x1, x2
 6821 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6822 %{
 6823   single_instruction;
 6824   dst    : EX1(write);
 6825   src1   : ISS(read);
 6826   src2   : ISS(read);
 6827   INS01  : ISS;
 6828   ALU    : EX1;
 6829 %}
 6830 
 6831 // Integer ALU reg-reg operation with extract
 6832 // As for _vshift above, but result generated in EX2
 6833 // Eg.  EXTR    x0, x1, x2, #N
 6834 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6835 %{
 6836   single_instruction;
 6837   dst    : EX2(write);
 6838   src1   : ISS(read);
 6839   src2   : ISS(read);
 6840   INS1   : ISS; // Can only dual issue as Instruction 1
 6841   ALU    : EX1;
 6842 %}
 6843 
 6844 // Integer ALU reg operation
 6845 // Eg.  NEG     x0, x1
 6846 pipe_class ialu_reg(iRegI dst, iRegI src)
 6847 %{
 6848   single_instruction;
 6849   dst    : EX2(write);
 6850   src    : EX1(read);
 6851   INS01  : ISS;
 6852   ALU    : EX2;
 6853 %}
 6854 
 6855 // Integer ALU reg mmediate operation
 6856 // Eg.  ADD     x0, x1, #N
 6857 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6858 %{
 6859   single_instruction;
 6860   dst    : EX2(write);
 6861   src1   : EX1(read);
 6862   INS01  : ISS;
 6863   ALU    : EX2;
 6864 %}
 6865 
 6866 // Integer ALU immediate operation (no source operands)
 6867 // Eg.  MOV     x0, #N
 6868 pipe_class ialu_imm(iRegI dst)
 6869 %{
 6870   single_instruction;
 6871   dst    : EX1(write);
 6872   INS01  : ISS;
 6873   ALU    : EX1;
 6874 %}
 6875 
 6876 //------- Compare operation -------------------------------
 6877 
 6878 // Compare reg-reg
 6879 // Eg.  CMP     x0, x1
 6880 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6881 %{
 6882   single_instruction;
 6883 //  fixed_latency(16);
 6884   cr     : EX2(write);
 6885   op1    : EX1(read);
 6886   op2    : EX1(read);
 6887   INS01  : ISS;
 6888   ALU    : EX2;
 6889 %}
 6890 
 6891 // Compare reg-reg
 6892 // Eg.  CMP     x0, #N
 6893 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6894 %{
 6895   single_instruction;
 6896 //  fixed_latency(16);
 6897   cr     : EX2(write);
 6898   op1    : EX1(read);
 6899   INS01  : ISS;
 6900   ALU    : EX2;
 6901 %}
 6902 
 6903 //------- Conditional instructions ------------------------
 6904 
 6905 // Conditional no operands
 6906 // Eg.  CSINC   x0, zr, zr, <cond>
 6907 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6908 %{
 6909   single_instruction;
 6910   cr     : EX1(read);
 6911   dst    : EX2(write);
 6912   INS01  : ISS;
 6913   ALU    : EX2;
 6914 %}
 6915 
 6916 // Conditional 2 operand
 6917 // EG.  CSEL    X0, X1, X2, <cond>
 6918 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6919 %{
 6920   single_instruction;
 6921   cr     : EX1(read);
 6922   src1   : EX1(read);
 6923   src2   : EX1(read);
 6924   dst    : EX2(write);
 6925   INS01  : ISS;
 6926   ALU    : EX2;
 6927 %}
 6928 
 6929 // Conditional 2 operand
 6930 // EG.  CSEL    X0, X1, X2, <cond>
 6931 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6932 %{
 6933   single_instruction;
 6934   cr     : EX1(read);
 6935   src    : EX1(read);
 6936   dst    : EX2(write);
 6937   INS01  : ISS;
 6938   ALU    : EX2;
 6939 %}
 6940 
 6941 //------- Multiply pipeline operations --------------------
 6942 
 6943 // Multiply reg-reg
 6944 // Eg.  MUL     w0, w1, w2
 6945 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6946 %{
 6947   single_instruction;
 6948   dst    : WR(write);
 6949   src1   : ISS(read);
 6950   src2   : ISS(read);
 6951   INS01  : ISS;
 6952   MAC    : WR;
 6953 %}
 6954 
 6955 // Multiply accumulate
 6956 // Eg.  MADD    w0, w1, w2, w3
 6957 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6958 %{
 6959   single_instruction;
 6960   dst    : WR(write);
 6961   src1   : ISS(read);
 6962   src2   : ISS(read);
 6963   src3   : ISS(read);
 6964   INS01  : ISS;
 6965   MAC    : WR;
 6966 %}
 6967 
 6968 // Eg.  MUL     w0, w1, w2
 6969 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6970 %{
 6971   single_instruction;
 6972   fixed_latency(3); // Maximum latency for 64 bit mul
 6973   dst    : WR(write);
 6974   src1   : ISS(read);
 6975   src2   : ISS(read);
 6976   INS01  : ISS;
 6977   MAC    : WR;
 6978 %}
 6979 
 6980 // Multiply accumulate
 6981 // Eg.  MADD    w0, w1, w2, w3
 6982 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6983 %{
 6984   single_instruction;
 6985   fixed_latency(3); // Maximum latency for 64 bit mul
 6986   dst    : WR(write);
 6987   src1   : ISS(read);
 6988   src2   : ISS(read);
 6989   src3   : ISS(read);
 6990   INS01  : ISS;
 6991   MAC    : WR;
 6992 %}
 6993 
 6994 //------- Divide pipeline operations --------------------
 6995 
 6996 // Eg.  SDIV    w0, w1, w2
 6997 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6998 %{
 6999   single_instruction;
 7000   fixed_latency(8); // Maximum latency for 32 bit divide
 7001   dst    : WR(write);
 7002   src1   : ISS(read);
 7003   src2   : ISS(read);
 7004   INS0   : ISS; // Can only dual issue as instruction 0
 7005   DIV    : WR;
 7006 %}
 7007 
 7008 // Eg.  SDIV    x0, x1, x2
 7009 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7010 %{
 7011   single_instruction;
 7012   fixed_latency(16); // Maximum latency for 64 bit divide
 7013   dst    : WR(write);
 7014   src1   : ISS(read);
 7015   src2   : ISS(read);
 7016   INS0   : ISS; // Can only dual issue as instruction 0
 7017   DIV    : WR;
 7018 %}
 7019 
 7020 //------- Load pipeline operations ------------------------
 7021 
 7022 // Load - prefetch
 7023 // Eg.  PFRM    <mem>
 7024 pipe_class iload_prefetch(memory mem)
 7025 %{
 7026   single_instruction;
 7027   mem    : ISS(read);
 7028   INS01  : ISS;
 7029   LDST   : WR;
 7030 %}
 7031 
 7032 // Load - reg, mem
 7033 // Eg.  LDR     x0, <mem>
 7034 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7035 %{
 7036   single_instruction;
 7037   dst    : WR(write);
 7038   mem    : ISS(read);
 7039   INS01  : ISS;
 7040   LDST   : WR;
 7041 %}
 7042 
 7043 // Load - reg, reg
 7044 // Eg.  LDR     x0, [sp, x1]
 7045 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7046 %{
 7047   single_instruction;
 7048   dst    : WR(write);
 7049   src    : ISS(read);
 7050   INS01  : ISS;
 7051   LDST   : WR;
 7052 %}
 7053 
 7054 //------- Store pipeline operations -----------------------
 7055 
 7056 // Store - zr, mem
 7057 // Eg.  STR     zr, <mem>
 7058 pipe_class istore_mem(memory mem)
 7059 %{
 7060   single_instruction;
 7061   mem    : ISS(read);
 7062   INS01  : ISS;
 7063   LDST   : WR;
 7064 %}
 7065 
 7066 // Store - reg, mem
 7067 // Eg.  STR     x0, <mem>
 7068 pipe_class istore_reg_mem(iRegI src, memory mem)
 7069 %{
 7070   single_instruction;
 7071   mem    : ISS(read);
 7072   src    : EX2(read);
 7073   INS01  : ISS;
 7074   LDST   : WR;
 7075 %}
 7076 
 7077 // Store - reg, reg
 7078 // Eg. STR      x0, [sp, x1]
 7079 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7080 %{
 7081   single_instruction;
 7082   dst    : ISS(read);
 7083   src    : EX2(read);
 7084   INS01  : ISS;
 7085   LDST   : WR;
 7086 %}
 7087 
 7088 //------- Store pipeline operations -----------------------
 7089 
 7090 // Branch
 7091 pipe_class pipe_branch()
 7092 %{
 7093   single_instruction;
 7094   INS01  : ISS;
 7095   BRANCH : EX1;
 7096 %}
 7097 
 7098 // Conditional branch
 7099 pipe_class pipe_branch_cond(rFlagsReg cr)
 7100 %{
 7101   single_instruction;
 7102   cr     : EX1(read);
 7103   INS01  : ISS;
 7104   BRANCH : EX1;
 7105 %}
 7106 
 7107 // Compare & Branch
 7108 // EG.  CBZ/CBNZ
 7109 pipe_class pipe_cmp_branch(iRegI op1)
 7110 %{
 7111   single_instruction;
 7112   op1    : EX1(read);
 7113   INS01  : ISS;
 7114   BRANCH : EX1;
 7115 %}
 7116 
 7117 //------- Synchronisation operations ----------------------
 7118 
 7119 // Any operation requiring serialization.
 7120 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7121 pipe_class pipe_serial()
 7122 %{
 7123   single_instruction;
 7124   force_serialization;
 7125   fixed_latency(16);
 7126   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7127   LDST   : WR;
 7128 %}
 7129 
 7130 // Generic big/slow expanded idiom - also serialized
 7131 pipe_class pipe_slow()
 7132 %{
 7133   instruction_count(10);
 7134   multiple_bundles;
 7135   force_serialization;
 7136   fixed_latency(16);
 7137   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7138   LDST   : WR;
 7139 %}
 7140 
 7141 // Empty pipeline class
 7142 pipe_class pipe_class_empty()
 7143 %{
 7144   single_instruction;
 7145   fixed_latency(0);
 7146 %}
 7147 
 7148 // Default pipeline class.
 7149 pipe_class pipe_class_default()
 7150 %{
 7151   single_instruction;
 7152   fixed_latency(2);
 7153 %}
 7154 
 7155 // Pipeline class for compares.
 7156 pipe_class pipe_class_compare()
 7157 %{
 7158   single_instruction;
 7159   fixed_latency(16);
 7160 %}
 7161 
 7162 // Pipeline class for memory operations.
 7163 pipe_class pipe_class_memory()
 7164 %{
 7165   single_instruction;
 7166   fixed_latency(16);
 7167 %}
 7168 
 7169 // Pipeline class for call.
 7170 pipe_class pipe_class_call()
 7171 %{
 7172   single_instruction;
 7173   fixed_latency(100);
 7174 %}
 7175 
 7176 // Define the class for the Nop node.
 7177 define %{
 7178    MachNop = pipe_class_empty;
 7179 %}
 7180 
 7181 %}
 7182 //----------INSTRUCTIONS-------------------------------------------------------
 7183 //
 7184 // match      -- States which machine-independent subtree may be replaced
 7185 //               by this instruction.
 7186 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7187 //               selection to identify a minimum cost tree of machine
 7188 //               instructions that matches a tree of machine-independent
 7189 //               instructions.
 7190 // format     -- A string providing the disassembly for this instruction.
 7191 //               The value of an instruction's operand may be inserted
 7192 //               by referring to it with a '$' prefix.
 7193 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7194 //               to within an encode class as $primary, $secondary, and $tertiary
 7195 //               rrspectively.  The primary opcode is commonly used to
 7196 //               indicate the type of machine instruction, while secondary
 7197 //               and tertiary are often used for prefix options or addressing
 7198 //               modes.
 7199 // ins_encode -- A list of encode classes with parameters. The encode class
 7200 //               name must have been defined in an 'enc_class' specification
 7201 //               in the encode section of the architecture description.
 7202 
 7203 // ============================================================================
 7204 // Memory (Load/Store) Instructions
 7205 
 7206 // Load Instructions
 7207 
 7208 // Load Byte (8 bit signed)
 7209 instruct loadB(iRegINoSp dst, memory1 mem)
 7210 %{
 7211   match(Set dst (LoadB mem));
 7212   predicate(!needs_acquiring_load(n));
 7213 
 7214   ins_cost(4 * INSN_COST);
 7215   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7216 
 7217   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7218 
 7219   ins_pipe(iload_reg_mem);
 7220 %}
 7221 
 7222 // Load Byte (8 bit signed) into long
 7223 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7224 %{
 7225   match(Set dst (ConvI2L (LoadB mem)));
 7226   predicate(!needs_acquiring_load(n->in(1)));
 7227 
 7228   ins_cost(4 * INSN_COST);
 7229   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7230 
 7231   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7232 
 7233   ins_pipe(iload_reg_mem);
 7234 %}
 7235 
 7236 // Load Byte (8 bit unsigned)
 7237 instruct loadUB(iRegINoSp dst, memory1 mem)
 7238 %{
 7239   match(Set dst (LoadUB mem));
 7240   predicate(!needs_acquiring_load(n));
 7241 
 7242   ins_cost(4 * INSN_COST);
 7243   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7244 
 7245   ins_encode(aarch64_enc_ldrb(dst, mem));
 7246 
 7247   ins_pipe(iload_reg_mem);
 7248 %}
 7249 
 7250 // Load Byte (8 bit unsigned) into long
 7251 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7252 %{
 7253   match(Set dst (ConvI2L (LoadUB mem)));
 7254   predicate(!needs_acquiring_load(n->in(1)));
 7255 
 7256   ins_cost(4 * INSN_COST);
 7257   format %{ "ldrb  $dst, $mem\t# byte" %}
 7258 
 7259   ins_encode(aarch64_enc_ldrb(dst, mem));
 7260 
 7261   ins_pipe(iload_reg_mem);
 7262 %}
 7263 
 7264 // Load Short (16 bit signed)
 7265 instruct loadS(iRegINoSp dst, memory2 mem)
 7266 %{
 7267   match(Set dst (LoadS mem));
 7268   predicate(!needs_acquiring_load(n));
 7269 
 7270   ins_cost(4 * INSN_COST);
 7271   format %{ "ldrshw  $dst, $mem\t# short" %}
 7272 
 7273   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7274 
 7275   ins_pipe(iload_reg_mem);
 7276 %}
 7277 
 7278 // Load Short (16 bit signed) into long
 7279 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7280 %{
 7281   match(Set dst (ConvI2L (LoadS mem)));
 7282   predicate(!needs_acquiring_load(n->in(1)));
 7283 
 7284   ins_cost(4 * INSN_COST);
 7285   format %{ "ldrsh  $dst, $mem\t# short" %}
 7286 
 7287   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7288 
 7289   ins_pipe(iload_reg_mem);
 7290 %}
 7291 
 7292 // Load Char (16 bit unsigned)
 7293 instruct loadUS(iRegINoSp dst, memory2 mem)
 7294 %{
 7295   match(Set dst (LoadUS mem));
 7296   predicate(!needs_acquiring_load(n));
 7297 
 7298   ins_cost(4 * INSN_COST);
 7299   format %{ "ldrh  $dst, $mem\t# short" %}
 7300 
 7301   ins_encode(aarch64_enc_ldrh(dst, mem));
 7302 
 7303   ins_pipe(iload_reg_mem);
 7304 %}
 7305 
 7306 // Load Short/Char (16 bit unsigned) into long
 7307 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7308 %{
 7309   match(Set dst (ConvI2L (LoadUS mem)));
 7310   predicate(!needs_acquiring_load(n->in(1)));
 7311 
 7312   ins_cost(4 * INSN_COST);
 7313   format %{ "ldrh  $dst, $mem\t# short" %}
 7314 
 7315   ins_encode(aarch64_enc_ldrh(dst, mem));
 7316 
 7317   ins_pipe(iload_reg_mem);
 7318 %}
 7319 
 7320 // Load Integer (32 bit signed)
 7321 instruct loadI(iRegINoSp dst, memory4 mem)
 7322 %{
 7323   match(Set dst (LoadI mem));
 7324   predicate(!needs_acquiring_load(n));
 7325 
 7326   ins_cost(4 * INSN_COST);
 7327   format %{ "ldrw  $dst, $mem\t# int" %}
 7328 
 7329   ins_encode(aarch64_enc_ldrw(dst, mem));
 7330 
 7331   ins_pipe(iload_reg_mem);
 7332 %}
 7333 
 7334 // Load Integer (32 bit signed) into long
 7335 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7336 %{
 7337   match(Set dst (ConvI2L (LoadI mem)));
 7338   predicate(!needs_acquiring_load(n->in(1)));
 7339 
 7340   ins_cost(4 * INSN_COST);
 7341   format %{ "ldrsw  $dst, $mem\t# int" %}
 7342 
 7343   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7344 
 7345   ins_pipe(iload_reg_mem);
 7346 %}
 7347 
 7348 // Load Integer (32 bit unsigned) into long
 7349 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7350 %{
 7351   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7352   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7353 
 7354   ins_cost(4 * INSN_COST);
 7355   format %{ "ldrw  $dst, $mem\t# int" %}
 7356 
 7357   ins_encode(aarch64_enc_ldrw(dst, mem));
 7358 
 7359   ins_pipe(iload_reg_mem);
 7360 %}
 7361 
 7362 // Load Long (64 bit signed)
 7363 instruct loadL(iRegLNoSp dst, memory8 mem)
 7364 %{
 7365   match(Set dst (LoadL mem));
 7366   predicate(!needs_acquiring_load(n));
 7367 
 7368   ins_cost(4 * INSN_COST);
 7369   format %{ "ldr  $dst, $mem\t# int" %}
 7370 
 7371   ins_encode(aarch64_enc_ldr(dst, mem));
 7372 
 7373   ins_pipe(iload_reg_mem);
 7374 %}
 7375 
 7376 // Load Range
 7377 instruct loadRange(iRegINoSp dst, memory4 mem)
 7378 %{
 7379   match(Set dst (LoadRange mem));
 7380 
 7381   ins_cost(4 * INSN_COST);
 7382   format %{ "ldrw  $dst, $mem\t# range" %}
 7383 
 7384   ins_encode(aarch64_enc_ldrw(dst, mem));
 7385 
 7386   ins_pipe(iload_reg_mem);
 7387 %}
 7388 
 7389 // Load Pointer
 7390 instruct loadP(iRegPNoSp dst, memory8 mem)
 7391 %{
 7392   match(Set dst (LoadP mem));
 7393   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7394 
 7395   ins_cost(4 * INSN_COST);
 7396   format %{ "ldr  $dst, $mem\t# ptr" %}
 7397 
 7398   ins_encode(aarch64_enc_ldr(dst, mem));
 7399 
 7400   ins_pipe(iload_reg_mem);
 7401 %}
 7402 
 7403 // Load Compressed Pointer
 7404 instruct loadN(iRegNNoSp dst, memory4 mem)
 7405 %{
 7406   match(Set dst (LoadN mem));
 7407   predicate(!needs_acquiring_load(n));
 7408 
 7409   ins_cost(4 * INSN_COST);
 7410   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7411 
 7412   ins_encode(aarch64_enc_ldrw(dst, mem));
 7413 
 7414   ins_pipe(iload_reg_mem);
 7415 %}
 7416 
 7417 // Load Klass Pointer
 7418 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7419 %{
 7420   match(Set dst (LoadKlass mem));
 7421   predicate(!needs_acquiring_load(n));
 7422 
 7423   ins_cost(4 * INSN_COST);
 7424   format %{ "ldr  $dst, $mem\t# class" %}
 7425 
 7426   ins_encode(aarch64_enc_ldr(dst, mem));
 7427 
 7428   ins_pipe(iload_reg_mem);
 7429 %}
 7430 
 7431 // Load Narrow Klass Pointer
 7432 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7433 %{
 7434   match(Set dst (LoadNKlass mem));
 7435   predicate(!needs_acquiring_load(n));
 7436 
 7437   ins_cost(4 * INSN_COST);
 7438   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7439 
 7440   ins_encode(aarch64_enc_ldrw(dst, mem));
 7441 
 7442   ins_pipe(iload_reg_mem);
 7443 %}
 7444 
 7445 // Load Float
 7446 instruct loadF(vRegF dst, memory4 mem)
 7447 %{
 7448   match(Set dst (LoadF mem));
 7449   predicate(!needs_acquiring_load(n));
 7450 
 7451   ins_cost(4 * INSN_COST);
 7452   format %{ "ldrs  $dst, $mem\t# float" %}
 7453 
 7454   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7455 
 7456   ins_pipe(pipe_class_memory);
 7457 %}
 7458 
 7459 // Load Double
 7460 instruct loadD(vRegD dst, memory8 mem)
 7461 %{
 7462   match(Set dst (LoadD mem));
 7463   predicate(!needs_acquiring_load(n));
 7464 
 7465   ins_cost(4 * INSN_COST);
 7466   format %{ "ldrd  $dst, $mem\t# double" %}
 7467 
 7468   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7469 
 7470   ins_pipe(pipe_class_memory);
 7471 %}
 7472 
 7473 
 7474 // Load Int Constant
 7475 instruct loadConI(iRegINoSp dst, immI src)
 7476 %{
 7477   match(Set dst src);
 7478 
 7479   ins_cost(INSN_COST);
 7480   format %{ "mov $dst, $src\t# int" %}
 7481 
 7482   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7483 
 7484   ins_pipe(ialu_imm);
 7485 %}
 7486 
 7487 // Load Long Constant
 7488 instruct loadConL(iRegLNoSp dst, immL src)
 7489 %{
 7490   match(Set dst src);
 7491 
 7492   ins_cost(INSN_COST);
 7493   format %{ "mov $dst, $src\t# long" %}
 7494 
 7495   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7496 
 7497   ins_pipe(ialu_imm);
 7498 %}
 7499 
 7500 // Load Pointer Constant
 7501 
 7502 instruct loadConP(iRegPNoSp dst, immP con)
 7503 %{
 7504   match(Set dst con);
 7505 
 7506   ins_cost(INSN_COST * 4);
 7507   format %{
 7508     "mov  $dst, $con\t# ptr\n\t"
 7509   %}
 7510 
 7511   ins_encode(aarch64_enc_mov_p(dst, con));
 7512 
 7513   ins_pipe(ialu_imm);
 7514 %}
 7515 
 7516 // Load Null Pointer Constant
 7517 
 7518 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7519 %{
 7520   match(Set dst con);
 7521 
 7522   ins_cost(INSN_COST);
 7523   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7524 
 7525   ins_encode(aarch64_enc_mov_p0(dst, con));
 7526 
 7527   ins_pipe(ialu_imm);
 7528 %}
 7529 
 7530 // Load Pointer Constant One
 7531 
 7532 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7533 %{
 7534   match(Set dst con);
 7535 
 7536   ins_cost(INSN_COST);
 7537   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7538 
 7539   ins_encode(aarch64_enc_mov_p1(dst, con));
 7540 
 7541   ins_pipe(ialu_imm);
 7542 %}
 7543 
 7544 // Load Byte Map Base Constant
 7545 
 7546 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7547 %{
 7548   match(Set dst con);
 7549 
 7550   ins_cost(INSN_COST);
 7551   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7552 
 7553   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7554 
 7555   ins_pipe(ialu_imm);
 7556 %}
 7557 
 7558 // Load Narrow Pointer Constant
 7559 
 7560 instruct loadConN(iRegNNoSp dst, immN con)
 7561 %{
 7562   match(Set dst con);
 7563 
 7564   ins_cost(INSN_COST * 4);
 7565   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7566 
 7567   ins_encode(aarch64_enc_mov_n(dst, con));
 7568 
 7569   ins_pipe(ialu_imm);
 7570 %}
 7571 
 7572 // Load Narrow Null Pointer Constant
 7573 
 7574 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7575 %{
 7576   match(Set dst con);
 7577 
 7578   ins_cost(INSN_COST);
 7579   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7580 
 7581   ins_encode(aarch64_enc_mov_n0(dst, con));
 7582 
 7583   ins_pipe(ialu_imm);
 7584 %}
 7585 
 7586 // Load Narrow Klass Constant
 7587 
 7588 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7589 %{
 7590   match(Set dst con);
 7591 
 7592   ins_cost(INSN_COST);
 7593   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7594 
 7595   ins_encode(aarch64_enc_mov_nk(dst, con));
 7596 
 7597   ins_pipe(ialu_imm);
 7598 %}
 7599 
 7600 // Load Packed Float Constant
 7601 
 7602 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7603   match(Set dst con);
 7604   ins_cost(INSN_COST * 4);
 7605   format %{ "fmovs  $dst, $con"%}
 7606   ins_encode %{
 7607     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7608   %}
 7609 
 7610   ins_pipe(fp_imm_s);
 7611 %}
 7612 
 7613 // Load Float Constant
 7614 
 7615 instruct loadConF(vRegF dst, immF con) %{
 7616   match(Set dst con);
 7617 
 7618   ins_cost(INSN_COST * 4);
 7619 
 7620   format %{
 7621     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7622   %}
 7623 
 7624   ins_encode %{
 7625     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7626   %}
 7627 
 7628   ins_pipe(fp_load_constant_s);
 7629 %}
 7630 
 7631 // Load Packed Double Constant
 7632 
 7633 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7634   match(Set dst con);
 7635   ins_cost(INSN_COST);
 7636   format %{ "fmovd  $dst, $con"%}
 7637   ins_encode %{
 7638     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7639   %}
 7640 
 7641   ins_pipe(fp_imm_d);
 7642 %}
 7643 
 7644 // Load Double Constant
 7645 
 7646 instruct loadConD(vRegD dst, immD con) %{
 7647   match(Set dst con);
 7648 
 7649   ins_cost(INSN_COST * 5);
 7650   format %{
 7651     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7652   %}
 7653 
 7654   ins_encode %{
 7655     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7656   %}
 7657 
 7658   ins_pipe(fp_load_constant_d);
 7659 %}
 7660 
 7661 // Store Instructions
 7662 
 7663 // Store CMS card-mark Immediate
 7664 instruct storeimmCM0(immI0 zero, memory1 mem)
 7665 %{
 7666   match(Set mem (StoreCM mem zero));
 7667 
 7668   ins_cost(INSN_COST);
 7669   format %{ "storestore (elided)\n\t"
 7670             "strb zr, $mem\t# byte" %}
 7671 
 7672   ins_encode(aarch64_enc_strb0(mem));
 7673 
 7674   ins_pipe(istore_mem);
 7675 %}
 7676 
 7677 // Store CMS card-mark Immediate with intervening StoreStore
 7678 // needed when using CMS with no conditional card marking
 7679 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7680 %{
 7681   match(Set mem (StoreCM mem zero));
 7682 
 7683   ins_cost(INSN_COST * 2);
 7684   format %{ "storestore\n\t"
 7685             "dmb ishst"
 7686             "\n\tstrb zr, $mem\t# byte" %}
 7687 
 7688   ins_encode(aarch64_enc_strb0_ordered(mem));
 7689 
 7690   ins_pipe(istore_mem);
 7691 %}
 7692 
 7693 // Store Byte
 7694 instruct storeB(iRegIorL2I src, memory1 mem)
 7695 %{
 7696   match(Set mem (StoreB mem src));
 7697   predicate(!needs_releasing_store(n));
 7698 
 7699   ins_cost(INSN_COST);
 7700   format %{ "strb  $src, $mem\t# byte" %}
 7701 
 7702   ins_encode(aarch64_enc_strb(src, mem));
 7703 
 7704   ins_pipe(istore_reg_mem);
 7705 %}
 7706 
 7707 
 7708 instruct storeimmB0(immI0 zero, memory1 mem)
 7709 %{
 7710   match(Set mem (StoreB mem zero));
 7711   predicate(!needs_releasing_store(n));
 7712 
 7713   ins_cost(INSN_COST);
 7714   format %{ "strb rscractch2, $mem\t# byte" %}
 7715 
 7716   ins_encode(aarch64_enc_strb0(mem));
 7717 
 7718   ins_pipe(istore_mem);
 7719 %}
 7720 
 7721 // Store Char/Short
 7722 instruct storeC(iRegIorL2I src, memory2 mem)
 7723 %{
 7724   match(Set mem (StoreC mem src));
 7725   predicate(!needs_releasing_store(n));
 7726 
 7727   ins_cost(INSN_COST);
 7728   format %{ "strh  $src, $mem\t# short" %}
 7729 
 7730   ins_encode(aarch64_enc_strh(src, mem));
 7731 
 7732   ins_pipe(istore_reg_mem);
 7733 %}
 7734 
 7735 instruct storeimmC0(immI0 zero, memory2 mem)
 7736 %{
 7737   match(Set mem (StoreC mem zero));
 7738   predicate(!needs_releasing_store(n));
 7739 
 7740   ins_cost(INSN_COST);
 7741   format %{ "strh  zr, $mem\t# short" %}
 7742 
 7743   ins_encode(aarch64_enc_strh0(mem));
 7744 
 7745   ins_pipe(istore_mem);
 7746 %}
 7747 
 7748 // Store Integer
 7749 
 7750 instruct storeI(iRegIorL2I src, memory4 mem)
 7751 %{
 7752   match(Set mem(StoreI mem src));
 7753   predicate(!needs_releasing_store(n));
 7754 
 7755   ins_cost(INSN_COST);
 7756   format %{ "strw  $src, $mem\t# int" %}
 7757 
 7758   ins_encode(aarch64_enc_strw(src, mem));
 7759 
 7760   ins_pipe(istore_reg_mem);
 7761 %}
 7762 
 7763 instruct storeimmI0(immI0 zero, memory4 mem)
 7764 %{
 7765   match(Set mem(StoreI mem zero));
 7766   predicate(!needs_releasing_store(n));
 7767 
 7768   ins_cost(INSN_COST);
 7769   format %{ "strw  zr, $mem\t# int" %}
 7770 
 7771   ins_encode(aarch64_enc_strw0(mem));
 7772 
 7773   ins_pipe(istore_mem);
 7774 %}
 7775 
 7776 // Store Long (64 bit signed)
 7777 instruct storeL(iRegL src, memory8 mem)
 7778 %{
 7779   match(Set mem (StoreL mem src));
 7780   predicate(!needs_releasing_store(n));
 7781 
 7782   ins_cost(INSN_COST);
 7783   format %{ "str  $src, $mem\t# int" %}
 7784 
 7785   ins_encode(aarch64_enc_str(src, mem));
 7786 
 7787   ins_pipe(istore_reg_mem);
 7788 %}
 7789 
 7790 // Store Long (64 bit signed)
 7791 instruct storeimmL0(immL0 zero, memory8 mem)
 7792 %{
 7793   match(Set mem (StoreL mem zero));
 7794   predicate(!needs_releasing_store(n));
 7795 
 7796   ins_cost(INSN_COST);
 7797   format %{ "str  zr, $mem\t# int" %}
 7798 
 7799   ins_encode(aarch64_enc_str0(mem));
 7800 
 7801   ins_pipe(istore_mem);
 7802 %}
 7803 
 7804 // Store Pointer
 7805 instruct storeP(iRegP src, memory8 mem)
 7806 %{
 7807   match(Set mem (StoreP mem src));
 7808   predicate(!needs_releasing_store(n));
 7809 
 7810   ins_cost(INSN_COST);
 7811   format %{ "str  $src, $mem\t# ptr" %}
 7812 
 7813   ins_encode(aarch64_enc_str(src, mem));
 7814 
 7815   ins_pipe(istore_reg_mem);
 7816 %}
 7817 
 7818 // Store Pointer
 7819 instruct storeimmP0(immP0 zero, memory8 mem)
 7820 %{
 7821   match(Set mem (StoreP mem zero));
 7822   predicate(!needs_releasing_store(n));
 7823 
 7824   ins_cost(INSN_COST);
 7825   format %{ "str zr, $mem\t# ptr" %}
 7826 
 7827   ins_encode(aarch64_enc_str0(mem));
 7828 
 7829   ins_pipe(istore_mem);
 7830 %}
 7831 
 7832 // Store Compressed Pointer
 7833 instruct storeN(iRegN src, memory4 mem)
 7834 %{
 7835   match(Set mem (StoreN mem src));
 7836   predicate(!needs_releasing_store(n));
 7837 
 7838   ins_cost(INSN_COST);
 7839   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7840 
 7841   ins_encode(aarch64_enc_strw(src, mem));
 7842 
 7843   ins_pipe(istore_reg_mem);
 7844 %}
 7845 
 7846 instruct storeImmN0(immN0 zero, memory4 mem)
 7847 %{
 7848   match(Set mem (StoreN mem zero));
 7849   predicate(!needs_releasing_store(n));
 7850 
 7851   ins_cost(INSN_COST);
 7852   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7853 
 7854   ins_encode(aarch64_enc_strw0(mem));
 7855 
 7856   ins_pipe(istore_mem);
 7857 %}
 7858 
 7859 // Store Float
 7860 instruct storeF(vRegF src, memory4 mem)
 7861 %{
 7862   match(Set mem (StoreF mem src));
 7863   predicate(!needs_releasing_store(n));
 7864 
 7865   ins_cost(INSN_COST);
 7866   format %{ "strs  $src, $mem\t# float" %}
 7867 
 7868   ins_encode( aarch64_enc_strs(src, mem) );
 7869 
 7870   ins_pipe(pipe_class_memory);
 7871 %}
 7872 
 7873 // TODO
 7874 // implement storeImmF0 and storeFImmPacked
 7875 
 7876 // Store Double
 7877 instruct storeD(vRegD src, memory8 mem)
 7878 %{
 7879   match(Set mem (StoreD mem src));
 7880   predicate(!needs_releasing_store(n));
 7881 
 7882   ins_cost(INSN_COST);
 7883   format %{ "strd  $src, $mem\t# double" %}
 7884 
 7885   ins_encode( aarch64_enc_strd(src, mem) );
 7886 
 7887   ins_pipe(pipe_class_memory);
 7888 %}
 7889 
 7890 // Store Compressed Klass Pointer
 7891 instruct storeNKlass(iRegN src, memory4 mem)
 7892 %{
 7893   predicate(!needs_releasing_store(n));
 7894   match(Set mem (StoreNKlass mem src));
 7895 
 7896   ins_cost(INSN_COST);
 7897   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7898 
 7899   ins_encode(aarch64_enc_strw(src, mem));
 7900 
 7901   ins_pipe(istore_reg_mem);
 7902 %}
 7903 
 7904 // TODO
 7905 // implement storeImmD0 and storeDImmPacked
 7906 
 7907 // prefetch instructions
 7908 // Must be safe to execute with invalid address (cannot fault).
 7909 
 7910 instruct prefetchalloc( memory8 mem ) %{
 7911   match(PrefetchAllocation mem);
 7912 
 7913   ins_cost(INSN_COST);
 7914   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7915 
 7916   ins_encode( aarch64_enc_prefetchw(mem) );
 7917 
 7918   ins_pipe(iload_prefetch);
 7919 %}
 7920 
 7921 //  ---------------- volatile loads and stores ----------------
 7922 
 7923 // Load Byte (8 bit signed)
 7924 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7925 %{
 7926   match(Set dst (LoadB mem));
 7927 
 7928   ins_cost(VOLATILE_REF_COST);
 7929   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7930 
 7931   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7932 
 7933   ins_pipe(pipe_serial);
 7934 %}
 7935 
 7936 // Load Byte (8 bit signed) into long
 7937 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7938 %{
 7939   match(Set dst (ConvI2L (LoadB mem)));
 7940 
 7941   ins_cost(VOLATILE_REF_COST);
 7942   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7943 
 7944   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7945 
 7946   ins_pipe(pipe_serial);
 7947 %}
 7948 
 7949 // Load Byte (8 bit unsigned)
 7950 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7951 %{
 7952   match(Set dst (LoadUB mem));
 7953 
 7954   ins_cost(VOLATILE_REF_COST);
 7955   format %{ "ldarb  $dst, $mem\t# byte" %}
 7956 
 7957   ins_encode(aarch64_enc_ldarb(dst, mem));
 7958 
 7959   ins_pipe(pipe_serial);
 7960 %}
 7961 
 7962 // Load Byte (8 bit unsigned) into long
 7963 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7964 %{
 7965   match(Set dst (ConvI2L (LoadUB mem)));
 7966 
 7967   ins_cost(VOLATILE_REF_COST);
 7968   format %{ "ldarb  $dst, $mem\t# byte" %}
 7969 
 7970   ins_encode(aarch64_enc_ldarb(dst, mem));
 7971 
 7972   ins_pipe(pipe_serial);
 7973 %}
 7974 
 7975 // Load Short (16 bit signed)
 7976 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7977 %{
 7978   match(Set dst (LoadS mem));
 7979 
 7980   ins_cost(VOLATILE_REF_COST);
 7981   format %{ "ldarshw  $dst, $mem\t# short" %}
 7982 
 7983   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7984 
 7985   ins_pipe(pipe_serial);
 7986 %}
 7987 
 7988 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7989 %{
 7990   match(Set dst (LoadUS mem));
 7991 
 7992   ins_cost(VOLATILE_REF_COST);
 7993   format %{ "ldarhw  $dst, $mem\t# short" %}
 7994 
 7995   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7996 
 7997   ins_pipe(pipe_serial);
 7998 %}
 7999 
 8000 // Load Short/Char (16 bit unsigned) into long
 8001 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8002 %{
 8003   match(Set dst (ConvI2L (LoadUS mem)));
 8004 
 8005   ins_cost(VOLATILE_REF_COST);
 8006   format %{ "ldarh  $dst, $mem\t# short" %}
 8007 
 8008   ins_encode(aarch64_enc_ldarh(dst, mem));
 8009 
 8010   ins_pipe(pipe_serial);
 8011 %}
 8012 
 8013 // Load Short/Char (16 bit signed) into long
 8014 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8015 %{
 8016   match(Set dst (ConvI2L (LoadS mem)));
 8017 
 8018   ins_cost(VOLATILE_REF_COST);
 8019   format %{ "ldarh  $dst, $mem\t# short" %}
 8020 
 8021   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8022 
 8023   ins_pipe(pipe_serial);
 8024 %}
 8025 
 8026 // Load Integer (32 bit signed)
 8027 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8028 %{
 8029   match(Set dst (LoadI mem));
 8030 
 8031   ins_cost(VOLATILE_REF_COST);
 8032   format %{ "ldarw  $dst, $mem\t# int" %}
 8033 
 8034   ins_encode(aarch64_enc_ldarw(dst, mem));
 8035 
 8036   ins_pipe(pipe_serial);
 8037 %}
 8038 
 8039 // Load Integer (32 bit unsigned) into long
 8040 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8041 %{
 8042   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8043 
 8044   ins_cost(VOLATILE_REF_COST);
 8045   format %{ "ldarw  $dst, $mem\t# int" %}
 8046 
 8047   ins_encode(aarch64_enc_ldarw(dst, mem));
 8048 
 8049   ins_pipe(pipe_serial);
 8050 %}
 8051 
 8052 // Load Long (64 bit signed)
 8053 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8054 %{
 8055   match(Set dst (LoadL mem));
 8056 
 8057   ins_cost(VOLATILE_REF_COST);
 8058   format %{ "ldar  $dst, $mem\t# int" %}
 8059 
 8060   ins_encode(aarch64_enc_ldar(dst, mem));
 8061 
 8062   ins_pipe(pipe_serial);
 8063 %}
 8064 
 8065 // Load Pointer
 8066 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8067 %{
 8068   match(Set dst (LoadP mem));
 8069   predicate(n->as_Load()->barrier_data() == 0);
 8070 
 8071   ins_cost(VOLATILE_REF_COST);
 8072   format %{ "ldar  $dst, $mem\t# ptr" %}
 8073 
 8074   ins_encode(aarch64_enc_ldar(dst, mem));
 8075 
 8076   ins_pipe(pipe_serial);
 8077 %}
 8078 
 8079 // Load Compressed Pointer
 8080 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8081 %{
 8082   match(Set dst (LoadN mem));
 8083 
 8084   ins_cost(VOLATILE_REF_COST);
 8085   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8086 
 8087   ins_encode(aarch64_enc_ldarw(dst, mem));
 8088 
 8089   ins_pipe(pipe_serial);
 8090 %}
 8091 
 8092 // Load Float
 8093 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8094 %{
 8095   match(Set dst (LoadF mem));
 8096 
 8097   ins_cost(VOLATILE_REF_COST);
 8098   format %{ "ldars  $dst, $mem\t# float" %}
 8099 
 8100   ins_encode( aarch64_enc_fldars(dst, mem) );
 8101 
 8102   ins_pipe(pipe_serial);
 8103 %}
 8104 
 8105 // Load Double
 8106 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8107 %{
 8108   match(Set dst (LoadD mem));
 8109 
 8110   ins_cost(VOLATILE_REF_COST);
 8111   format %{ "ldard  $dst, $mem\t# double" %}
 8112 
 8113   ins_encode( aarch64_enc_fldard(dst, mem) );
 8114 
 8115   ins_pipe(pipe_serial);
 8116 %}
 8117 
 8118 // Store Byte
 8119 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8120 %{
 8121   match(Set mem (StoreB mem src));
 8122 
 8123   ins_cost(VOLATILE_REF_COST);
 8124   format %{ "stlrb  $src, $mem\t# byte" %}
 8125 
 8126   ins_encode(aarch64_enc_stlrb(src, mem));
 8127 
 8128   ins_pipe(pipe_class_memory);
 8129 %}
 8130 
 8131 // Store Char/Short
 8132 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8133 %{
 8134   match(Set mem (StoreC mem src));
 8135 
 8136   ins_cost(VOLATILE_REF_COST);
 8137   format %{ "stlrh  $src, $mem\t# short" %}
 8138 
 8139   ins_encode(aarch64_enc_stlrh(src, mem));
 8140 
 8141   ins_pipe(pipe_class_memory);
 8142 %}
 8143 
 8144 // Store Integer
 8145 
 8146 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8147 %{
 8148   match(Set mem(StoreI mem src));
 8149 
 8150   ins_cost(VOLATILE_REF_COST);
 8151   format %{ "stlrw  $src, $mem\t# int" %}
 8152 
 8153   ins_encode(aarch64_enc_stlrw(src, mem));
 8154 
 8155   ins_pipe(pipe_class_memory);
 8156 %}
 8157 
 8158 // Store Long (64 bit signed)
 8159 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8160 %{
 8161   match(Set mem (StoreL mem src));
 8162 
 8163   ins_cost(VOLATILE_REF_COST);
 8164   format %{ "stlr  $src, $mem\t# int" %}
 8165 
 8166   ins_encode(aarch64_enc_stlr(src, mem));
 8167 
 8168   ins_pipe(pipe_class_memory);
 8169 %}
 8170 
 8171 // Store Pointer
 8172 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8173 %{
 8174   match(Set mem (StoreP mem src));
 8175 
 8176   ins_cost(VOLATILE_REF_COST);
 8177   format %{ "stlr  $src, $mem\t# ptr" %}
 8178 
 8179   ins_encode(aarch64_enc_stlr(src, mem));
 8180 
 8181   ins_pipe(pipe_class_memory);
 8182 %}
 8183 
 8184 // Store Compressed Pointer
 8185 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8186 %{
 8187   match(Set mem (StoreN mem src));
 8188 
 8189   ins_cost(VOLATILE_REF_COST);
 8190   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8191 
 8192   ins_encode(aarch64_enc_stlrw(src, mem));
 8193 
 8194   ins_pipe(pipe_class_memory);
 8195 %}
 8196 
 8197 // Store Float
 8198 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8199 %{
 8200   match(Set mem (StoreF mem src));
 8201 
 8202   ins_cost(VOLATILE_REF_COST);
 8203   format %{ "stlrs  $src, $mem\t# float" %}
 8204 
 8205   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8206 
 8207   ins_pipe(pipe_class_memory);
 8208 %}
 8209 
 8210 // TODO
 8211 // implement storeImmF0 and storeFImmPacked
 8212 
 8213 // Store Double
 8214 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8215 %{
 8216   match(Set mem (StoreD mem src));
 8217 
 8218   ins_cost(VOLATILE_REF_COST);
 8219   format %{ "stlrd  $src, $mem\t# double" %}
 8220 
 8221   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8222 
 8223   ins_pipe(pipe_class_memory);
 8224 %}
 8225 
 8226 //  ---------------- end of volatile loads and stores ----------------
 8227 
 8228 instruct cacheWB(indirect addr)
 8229 %{
 8230   predicate(VM_Version::supports_data_cache_line_flush());
 8231   match(CacheWB addr);
 8232 
 8233   ins_cost(100);
 8234   format %{"cache wb $addr" %}
 8235   ins_encode %{
 8236     assert($addr->index_position() < 0, "should be");
 8237     assert($addr$$disp == 0, "should be");
 8238     __ cache_wb(Address($addr$$base$$Register, 0));
 8239   %}
 8240   ins_pipe(pipe_slow); // XXX
 8241 %}
 8242 
 8243 instruct cacheWBPreSync()
 8244 %{
 8245   predicate(VM_Version::supports_data_cache_line_flush());
 8246   match(CacheWBPreSync);
 8247 
 8248   ins_cost(100);
 8249   format %{"cache wb presync" %}
 8250   ins_encode %{
 8251     __ cache_wbsync(true);
 8252   %}
 8253   ins_pipe(pipe_slow); // XXX
 8254 %}
 8255 
 8256 instruct cacheWBPostSync()
 8257 %{
 8258   predicate(VM_Version::supports_data_cache_line_flush());
 8259   match(CacheWBPostSync);
 8260 
 8261   ins_cost(100);
 8262   format %{"cache wb postsync" %}
 8263   ins_encode %{
 8264     __ cache_wbsync(false);
 8265   %}
 8266   ins_pipe(pipe_slow); // XXX
 8267 %}
 8268 
 8269 // ============================================================================
 8270 // BSWAP Instructions
 8271 
 8272 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8273   match(Set dst (ReverseBytesI src));
 8274 
 8275   ins_cost(INSN_COST);
 8276   format %{ "revw  $dst, $src" %}
 8277 
 8278   ins_encode %{
 8279     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8280   %}
 8281 
 8282   ins_pipe(ialu_reg);
 8283 %}
 8284 
 8285 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8286   match(Set dst (ReverseBytesL src));
 8287 
 8288   ins_cost(INSN_COST);
 8289   format %{ "rev  $dst, $src" %}
 8290 
 8291   ins_encode %{
 8292     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8293   %}
 8294 
 8295   ins_pipe(ialu_reg);
 8296 %}
 8297 
 8298 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8299   match(Set dst (ReverseBytesUS src));
 8300 
 8301   ins_cost(INSN_COST);
 8302   format %{ "rev16w  $dst, $src" %}
 8303 
 8304   ins_encode %{
 8305     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8306   %}
 8307 
 8308   ins_pipe(ialu_reg);
 8309 %}
 8310 
 8311 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8312   match(Set dst (ReverseBytesS src));
 8313 
 8314   ins_cost(INSN_COST);
 8315   format %{ "rev16w  $dst, $src\n\t"
 8316             "sbfmw $dst, $dst, #0, #15" %}
 8317 
 8318   ins_encode %{
 8319     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8320     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8321   %}
 8322 
 8323   ins_pipe(ialu_reg);
 8324 %}
 8325 
 8326 // ============================================================================
 8327 // Zero Count Instructions
 8328 
 8329 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8330   match(Set dst (CountLeadingZerosI src));
 8331 
 8332   ins_cost(INSN_COST);
 8333   format %{ "clzw  $dst, $src" %}
 8334   ins_encode %{
 8335     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8336   %}
 8337 
 8338   ins_pipe(ialu_reg);
 8339 %}
 8340 
 8341 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8342   match(Set dst (CountLeadingZerosL src));
 8343 
 8344   ins_cost(INSN_COST);
 8345   format %{ "clz   $dst, $src" %}
 8346   ins_encode %{
 8347     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8348   %}
 8349 
 8350   ins_pipe(ialu_reg);
 8351 %}
 8352 
 8353 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8354   match(Set dst (CountTrailingZerosI src));
 8355 
 8356   ins_cost(INSN_COST * 2);
 8357   format %{ "rbitw  $dst, $src\n\t"
 8358             "clzw   $dst, $dst" %}
 8359   ins_encode %{
 8360     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8361     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8362   %}
 8363 
 8364   ins_pipe(ialu_reg);
 8365 %}
 8366 
 8367 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8368   match(Set dst (CountTrailingZerosL src));
 8369 
 8370   ins_cost(INSN_COST * 2);
 8371   format %{ "rbit   $dst, $src\n\t"
 8372             "clz    $dst, $dst" %}
 8373   ins_encode %{
 8374     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8375     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8376   %}
 8377 
 8378   ins_pipe(ialu_reg);
 8379 %}
 8380 
 8381 //---------- Population Count Instructions -------------------------------------
 8382 //
 8383 
 8384 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8385   predicate(UsePopCountInstruction);
 8386   match(Set dst (PopCountI src));
 8387   effect(TEMP tmp);
 8388   ins_cost(INSN_COST * 13);
 8389 
 8390   format %{ "movw   $src, $src\n\t"
 8391             "mov    $tmp, $src\t# vector (1D)\n\t"
 8392             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8393             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8394             "mov    $dst, $tmp\t# vector (1D)" %}
 8395   ins_encode %{
 8396     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8397     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8398     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8399     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8400     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8401   %}
 8402 
 8403   ins_pipe(pipe_class_default);
 8404 %}
 8405 
 8406 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8407   predicate(UsePopCountInstruction);
 8408   match(Set dst (PopCountI (LoadI mem)));
 8409   effect(TEMP tmp);
 8410   ins_cost(INSN_COST * 13);
 8411 
 8412   format %{ "ldrs   $tmp, $mem\n\t"
 8413             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8414             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8415             "mov    $dst, $tmp\t# vector (1D)" %}
 8416   ins_encode %{
 8417     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8418     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8419               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8420     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8421     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8422     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8423   %}
 8424 
 8425   ins_pipe(pipe_class_default);
 8426 %}
 8427 
 8428 // Note: Long.bitCount(long) returns an int.
 8429 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8430   predicate(UsePopCountInstruction);
 8431   match(Set dst (PopCountL src));
 8432   effect(TEMP tmp);
 8433   ins_cost(INSN_COST * 13);
 8434 
 8435   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8436             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8437             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8438             "mov    $dst, $tmp\t# vector (1D)" %}
 8439   ins_encode %{
 8440     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8441     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8442     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8443     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8444   %}
 8445 
 8446   ins_pipe(pipe_class_default);
 8447 %}
 8448 
 8449 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8450   predicate(UsePopCountInstruction);
 8451   match(Set dst (PopCountL (LoadL mem)));
 8452   effect(TEMP tmp);
 8453   ins_cost(INSN_COST * 13);
 8454 
 8455   format %{ "ldrd   $tmp, $mem\n\t"
 8456             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8457             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8458             "mov    $dst, $tmp\t# vector (1D)" %}
 8459   ins_encode %{
 8460     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8461     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8462               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8463     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8464     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8465     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8466   %}
 8467 
 8468   ins_pipe(pipe_class_default);
 8469 %}
 8470 
 8471 // ============================================================================
 8472 // MemBar Instruction
 8473 
 8474 instruct load_fence() %{
 8475   match(LoadFence);
 8476   ins_cost(VOLATILE_REF_COST);
 8477 
 8478   format %{ "load_fence" %}
 8479 
 8480   ins_encode %{
 8481     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8482   %}
 8483   ins_pipe(pipe_serial);
 8484 %}
 8485 
 8486 instruct unnecessary_membar_acquire() %{
 8487   predicate(unnecessary_acquire(n));
 8488   match(MemBarAcquire);
 8489   ins_cost(0);
 8490 
 8491   format %{ "membar_acquire (elided)" %}
 8492 
 8493   ins_encode %{
 8494     __ block_comment("membar_acquire (elided)");
 8495   %}
 8496 
 8497   ins_pipe(pipe_class_empty);
 8498 %}
 8499 
 8500 instruct membar_acquire() %{
 8501   match(MemBarAcquire);
 8502   ins_cost(VOLATILE_REF_COST);
 8503 
 8504   format %{ "membar_acquire\n\t"
 8505             "dmb ish" %}
 8506 
 8507   ins_encode %{
 8508     __ block_comment("membar_acquire");
 8509     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8510   %}
 8511 
 8512   ins_pipe(pipe_serial);
 8513 %}
 8514 
 8515 
 8516 instruct membar_acquire_lock() %{
 8517   match(MemBarAcquireLock);
 8518   ins_cost(VOLATILE_REF_COST);
 8519 
 8520   format %{ "membar_acquire_lock (elided)" %}
 8521 
 8522   ins_encode %{
 8523     __ block_comment("membar_acquire_lock (elided)");
 8524   %}
 8525 
 8526   ins_pipe(pipe_serial);
 8527 %}
 8528 
 8529 instruct store_fence() %{
 8530   match(StoreFence);
 8531   ins_cost(VOLATILE_REF_COST);
 8532 
 8533   format %{ "store_fence" %}
 8534 
 8535   ins_encode %{
 8536     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8537   %}
 8538   ins_pipe(pipe_serial);
 8539 %}
 8540 
 8541 instruct unnecessary_membar_release() %{
 8542   predicate(unnecessary_release(n));
 8543   match(MemBarRelease);
 8544   ins_cost(0);
 8545 
 8546   format %{ "membar_release (elided)" %}
 8547 
 8548   ins_encode %{
 8549     __ block_comment("membar_release (elided)");
 8550   %}
 8551   ins_pipe(pipe_serial);
 8552 %}
 8553 
 8554 instruct membar_release() %{
 8555   match(MemBarRelease);
 8556   ins_cost(VOLATILE_REF_COST);
 8557 
 8558   format %{ "membar_release\n\t"
 8559             "dmb ish" %}
 8560 
 8561   ins_encode %{
 8562     __ block_comment("membar_release");
 8563     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8564   %}
 8565   ins_pipe(pipe_serial);
 8566 %}
 8567 
 8568 instruct membar_storestore() %{
 8569   match(MemBarStoreStore);
 8570   ins_cost(VOLATILE_REF_COST);
 8571 
 8572   format %{ "MEMBAR-store-store" %}
 8573 
 8574   ins_encode %{
 8575     __ membar(Assembler::StoreStore);
 8576   %}
 8577   ins_pipe(pipe_serial);
 8578 %}
 8579 
 8580 instruct membar_release_lock() %{
 8581   match(MemBarReleaseLock);
 8582   ins_cost(VOLATILE_REF_COST);
 8583 
 8584   format %{ "membar_release_lock (elided)" %}
 8585 
 8586   ins_encode %{
 8587     __ block_comment("membar_release_lock (elided)");
 8588   %}
 8589 
 8590   ins_pipe(pipe_serial);
 8591 %}
 8592 
 8593 instruct unnecessary_membar_volatile() %{
 8594   predicate(unnecessary_volatile(n));
 8595   match(MemBarVolatile);
 8596   ins_cost(0);
 8597 
 8598   format %{ "membar_volatile (elided)" %}
 8599 
 8600   ins_encode %{
 8601     __ block_comment("membar_volatile (elided)");
 8602   %}
 8603 
 8604   ins_pipe(pipe_serial);
 8605 %}
 8606 
 8607 instruct membar_volatile() %{
 8608   match(MemBarVolatile);
 8609   ins_cost(VOLATILE_REF_COST*100);
 8610 
 8611   format %{ "membar_volatile\n\t"
 8612              "dmb ish"%}
 8613 
 8614   ins_encode %{
 8615     __ block_comment("membar_volatile");
 8616     __ membar(Assembler::StoreLoad);
 8617   %}
 8618 
 8619   ins_pipe(pipe_serial);
 8620 %}
 8621 
 8622 // ============================================================================
 8623 // Cast/Convert Instructions
 8624 
 8625 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8626   match(Set dst (CastX2P src));
 8627 
 8628   ins_cost(INSN_COST);
 8629   format %{ "mov $dst, $src\t# long -> ptr" %}
 8630 
 8631   ins_encode %{
 8632     if ($dst$$reg != $src$$reg) {
 8633       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8634     }
 8635   %}
 8636 
 8637   ins_pipe(ialu_reg);
 8638 %}
 8639 
 8640 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8641   match(Set dst (CastP2X src));
 8642 
 8643   ins_cost(INSN_COST);
 8644   format %{ "mov $dst, $src\t# ptr -> long" %}
 8645 
 8646   ins_encode %{
 8647     if ($dst$$reg != $src$$reg) {
 8648       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8649     }
 8650   %}
 8651 
 8652   ins_pipe(ialu_reg);
 8653 %}
 8654 
 8655 // Convert oop into int for vectors alignment masking
 8656 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8657   match(Set dst (ConvL2I (CastP2X src)));
 8658 
 8659   ins_cost(INSN_COST);
 8660   format %{ "movw $dst, $src\t# ptr -> int" %}
 8661   ins_encode %{
 8662     __ movw($dst$$Register, $src$$Register);
 8663   %}
 8664 
 8665   ins_pipe(ialu_reg);
 8666 %}
 8667 
 8668 // Convert compressed oop into int for vectors alignment masking
 8669 // in case of 32bit oops (heap < 4Gb).
 8670 instruct convN2I(iRegINoSp dst, iRegN src)
 8671 %{
 8672   predicate(CompressedOops::shift() == 0);
 8673   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8674 
 8675   ins_cost(INSN_COST);
 8676   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8677   ins_encode %{
 8678     __ movw($dst$$Register, $src$$Register);
 8679   %}
 8680 
 8681   ins_pipe(ialu_reg);
 8682 %}
 8683 
 8684 
 8685 // Convert oop pointer into compressed form
 8686 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8687   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8688   match(Set dst (EncodeP src));
 8689   effect(KILL cr);
 8690   ins_cost(INSN_COST * 3);
 8691   format %{ "encode_heap_oop $dst, $src" %}
 8692   ins_encode %{
 8693     Register s = $src$$Register;
 8694     Register d = $dst$$Register;
 8695     __ encode_heap_oop(d, s);
 8696   %}
 8697   ins_pipe(ialu_reg);
 8698 %}
 8699 
 8700 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8701   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8702   match(Set dst (EncodeP src));
 8703   ins_cost(INSN_COST * 3);
 8704   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8705   ins_encode %{
 8706     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8707   %}
 8708   ins_pipe(ialu_reg);
 8709 %}
 8710 
 8711 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8712   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8713             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8714   match(Set dst (DecodeN src));
 8715   ins_cost(INSN_COST * 3);
 8716   format %{ "decode_heap_oop $dst, $src" %}
 8717   ins_encode %{
 8718     Register s = $src$$Register;
 8719     Register d = $dst$$Register;
 8720     __ decode_heap_oop(d, s);
 8721   %}
 8722   ins_pipe(ialu_reg);
 8723 %}
 8724 
 8725 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8726   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8727             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8728   match(Set dst (DecodeN src));
 8729   ins_cost(INSN_COST * 3);
 8730   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8731   ins_encode %{
 8732     Register s = $src$$Register;
 8733     Register d = $dst$$Register;
 8734     __ decode_heap_oop_not_null(d, s);
 8735   %}
 8736   ins_pipe(ialu_reg);
 8737 %}
 8738 
 8739 // n.b. AArch64 implementations of encode_klass_not_null and
 8740 // decode_klass_not_null do not modify the flags register so, unlike
 8741 // Intel, we don't kill CR as a side effect here
 8742 
 8743 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8744   match(Set dst (EncodePKlass src));
 8745 
 8746   ins_cost(INSN_COST * 3);
 8747   format %{ "encode_klass_not_null $dst,$src" %}
 8748 
 8749   ins_encode %{
 8750     Register src_reg = as_Register($src$$reg);
 8751     Register dst_reg = as_Register($dst$$reg);
 8752     __ encode_klass_not_null(dst_reg, src_reg);
 8753   %}
 8754 
 8755    ins_pipe(ialu_reg);
 8756 %}
 8757 
 8758 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8759   match(Set dst (DecodeNKlass src));
 8760 
 8761   ins_cost(INSN_COST * 3);
 8762   format %{ "decode_klass_not_null $dst,$src" %}
 8763 
 8764   ins_encode %{
 8765     Register src_reg = as_Register($src$$reg);
 8766     Register dst_reg = as_Register($dst$$reg);
 8767     if (dst_reg != src_reg) {
 8768       __ decode_klass_not_null(dst_reg, src_reg);
 8769     } else {
 8770       __ decode_klass_not_null(dst_reg);
 8771     }
 8772   %}
 8773 
 8774    ins_pipe(ialu_reg);
 8775 %}
 8776 
 8777 instruct checkCastPP(iRegPNoSp dst)
 8778 %{
 8779   match(Set dst (CheckCastPP dst));
 8780 
 8781   size(0);
 8782   format %{ "# checkcastPP of $dst" %}
 8783   ins_encode(/* empty encoding */);
 8784   ins_pipe(pipe_class_empty);
 8785 %}
 8786 
 8787 instruct castPP(iRegPNoSp dst)
 8788 %{
 8789   match(Set dst (CastPP dst));
 8790 
 8791   size(0);
 8792   format %{ "# castPP of $dst" %}
 8793   ins_encode(/* empty encoding */);
 8794   ins_pipe(pipe_class_empty);
 8795 %}
 8796 
 8797 instruct castII(iRegI dst)
 8798 %{
 8799   match(Set dst (CastII dst));
 8800 
 8801   size(0);
 8802   format %{ "# castII of $dst" %}
 8803   ins_encode(/* empty encoding */);
 8804   ins_cost(0);
 8805   ins_pipe(pipe_class_empty);
 8806 %}
 8807 
 8808 instruct castLL(iRegL dst)
 8809 %{
 8810   match(Set dst (CastLL dst));
 8811 
 8812   size(0);
 8813   format %{ "# castLL of $dst" %}
 8814   ins_encode(/* empty encoding */);
 8815   ins_cost(0);
 8816   ins_pipe(pipe_class_empty);
 8817 %}
 8818 
 8819 instruct castFF(vRegF dst)
 8820 %{
 8821   match(Set dst (CastFF dst));
 8822 
 8823   size(0);
 8824   format %{ "# castFF of $dst" %}
 8825   ins_encode(/* empty encoding */);
 8826   ins_cost(0);
 8827   ins_pipe(pipe_class_empty);
 8828 %}
 8829 
 8830 instruct castDD(vRegD dst)
 8831 %{
 8832   match(Set dst (CastDD dst));
 8833 
 8834   size(0);
 8835   format %{ "# castDD of $dst" %}
 8836   ins_encode(/* empty encoding */);
 8837   ins_cost(0);
 8838   ins_pipe(pipe_class_empty);
 8839 %}
 8840 
 8841 instruct castVVD(vecD dst)
 8842 %{
 8843   match(Set dst (CastVV dst));
 8844 
 8845   size(0);
 8846   format %{ "# castVV of $dst" %}
 8847   ins_encode(/* empty encoding */);
 8848   ins_cost(0);
 8849   ins_pipe(pipe_class_empty);
 8850 %}
 8851 
 8852 instruct castVVX(vecX dst)
 8853 %{
 8854   match(Set dst (CastVV dst));
 8855 
 8856   size(0);
 8857   format %{ "# castVV of $dst" %}
 8858   ins_encode(/* empty encoding */);
 8859   ins_cost(0);
 8860   ins_pipe(pipe_class_empty);
 8861 %}
 8862 
 8863 instruct castVV(vReg dst)
 8864 %{
 8865   match(Set dst (CastVV dst));
 8866 
 8867   size(0);
 8868   format %{ "# castVV of $dst" %}
 8869   ins_encode(/* empty encoding */);
 8870   ins_cost(0);
 8871   ins_pipe(pipe_class_empty);
 8872 %}
 8873 
 8874 // ============================================================================
 8875 // Atomic operation instructions
 8876 //
 8877 // Intel and SPARC both implement Ideal Node LoadPLocked and
 8878 // Store{PIL}Conditional instructions using a normal load for the
 8879 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 8880 //
 8881 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 8882 // pair to lock object allocations from Eden space when not using
 8883 // TLABs.
 8884 //
 8885 // There does not appear to be a Load{IL}Locked Ideal Node and the
 8886 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 8887 // and to use StoreIConditional only for 32-bit and StoreLConditional
 8888 // only for 64-bit.
 8889 //
 8890 // We implement LoadPLocked and StorePLocked instructions using,
 8891 // respectively the AArch64 hw load-exclusive and store-conditional
 8892 // instructions. Whereas we must implement each of
 8893 // Store{IL}Conditional using a CAS which employs a pair of
 8894 // instructions comprising a load-exclusive followed by a
 8895 // store-conditional.
 8896 
 8897 
 8898 // Locked-load (linked load) of the current heap-top
 8899 // used when updating the eden heap top
 8900 // implemented using ldaxr on AArch64
 8901 
 8902 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 8903 %{
 8904   match(Set dst (LoadPLocked mem));
 8905 
 8906   ins_cost(VOLATILE_REF_COST);
 8907 
 8908   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 8909 
 8910   ins_encode(aarch64_enc_ldaxr(dst, mem));
 8911 
 8912   ins_pipe(pipe_serial);
 8913 %}
 8914 
 8915 // Conditional-store of the updated heap-top.
 8916 // Used during allocation of the shared heap.
 8917 // Sets flag (EQ) on success.
 8918 // implemented using stlxr on AArch64.
 8919 
 8920 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 8921 %{
 8922   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 8923 
 8924   ins_cost(VOLATILE_REF_COST);
 8925 
 8926  // TODO
 8927  // do we need to do a store-conditional release or can we just use a
 8928  // plain store-conditional?
 8929 
 8930   format %{
 8931     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 8932     "cmpw rscratch1, zr\t# EQ on successful write"
 8933   %}
 8934 
 8935   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 8936 
 8937   ins_pipe(pipe_serial);
 8938 %}
 8939 
 8940 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 8941 %{
 8942   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 8943 
 8944   ins_cost(VOLATILE_REF_COST);
 8945 
 8946   format %{
 8947     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 8948     "cmpw rscratch1, zr\t# EQ on successful write"
 8949   %}
 8950 
 8951   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 8952 
 8953   ins_pipe(pipe_slow);
 8954 %}
 8955 
 8956 // storeIConditional also has acquire semantics, for no better reason
 8957 // than matching storeLConditional.  At the time of writing this
 8958 // comment storeIConditional was not used anywhere by AArch64.
 8959 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 8960 %{
 8961   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 8962 
 8963   ins_cost(VOLATILE_REF_COST);
 8964 
 8965   format %{
 8966     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 8967     "cmpw rscratch1, zr\t# EQ on successful write"
 8968   %}
 8969 
 8970   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 8971 
 8972   ins_pipe(pipe_slow);
 8973 %}
 8974 
 8975 // standard CompareAndSwapX when we are using barriers
 8976 // these have higher priority than the rules selected by a predicate
 8977 
 8978 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8979 // can't match them
 8980 
 8981 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8982 
 8983   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8984   ins_cost(2 * VOLATILE_REF_COST);
 8985 
 8986   effect(KILL cr);
 8987 
 8988   format %{
 8989     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8990     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8991   %}
 8992 
 8993   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8994             aarch64_enc_cset_eq(res));
 8995 
 8996   ins_pipe(pipe_slow);
 8997 %}
 8998 
 8999 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9000 
 9001   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9002   ins_cost(2 * VOLATILE_REF_COST);
 9003 
 9004   effect(KILL cr);
 9005 
 9006   format %{
 9007     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9008     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9009   %}
 9010 
 9011   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9012             aarch64_enc_cset_eq(res));
 9013 
 9014   ins_pipe(pipe_slow);
 9015 %}
 9016 
 9017 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9018 
 9019   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9020   ins_cost(2 * VOLATILE_REF_COST);
 9021 
 9022   effect(KILL cr);
 9023 
 9024  format %{
 9025     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9026     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9027  %}
 9028 
 9029  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9030             aarch64_enc_cset_eq(res));
 9031 
 9032   ins_pipe(pipe_slow);
 9033 %}
 9034 
 9035 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9036 
 9037   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9038   ins_cost(2 * VOLATILE_REF_COST);
 9039 
 9040   effect(KILL cr);
 9041 
 9042  format %{
 9043     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9044     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9045  %}
 9046 
 9047  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9048             aarch64_enc_cset_eq(res));
 9049 
 9050   ins_pipe(pipe_slow);
 9051 %}
 9052 
 9053 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9054 
 9055   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9056   predicate(n->as_LoadStore()->barrier_data() == 0);
 9057   ins_cost(2 * VOLATILE_REF_COST);
 9058 
 9059   effect(KILL cr);
 9060 
 9061  format %{
 9062     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9063     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9064  %}
 9065 
 9066  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9067             aarch64_enc_cset_eq(res));
 9068 
 9069   ins_pipe(pipe_slow);
 9070 %}
 9071 
 9072 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9073 
 9074   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9075   ins_cost(2 * VOLATILE_REF_COST);
 9076 
 9077   effect(KILL cr);
 9078 
 9079  format %{
 9080     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9081     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9082  %}
 9083 
 9084  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9085             aarch64_enc_cset_eq(res));
 9086 
 9087   ins_pipe(pipe_slow);
 9088 %}
 9089 
 9090 // alternative CompareAndSwapX when we are eliding barriers
 9091 
 9092 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9093 
 9094   predicate(needs_acquiring_load_exclusive(n));
 9095   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9096   ins_cost(VOLATILE_REF_COST);
 9097 
 9098   effect(KILL cr);
 9099 
 9100   format %{
 9101     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9102     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9103   %}
 9104 
 9105   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9106             aarch64_enc_cset_eq(res));
 9107 
 9108   ins_pipe(pipe_slow);
 9109 %}
 9110 
 9111 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9112 
 9113   predicate(needs_acquiring_load_exclusive(n));
 9114   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9115   ins_cost(VOLATILE_REF_COST);
 9116 
 9117   effect(KILL cr);
 9118 
 9119   format %{
 9120     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9121     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9122   %}
 9123 
 9124   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9125             aarch64_enc_cset_eq(res));
 9126 
 9127   ins_pipe(pipe_slow);
 9128 %}
 9129 
 9130 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9131 
 9132   predicate(needs_acquiring_load_exclusive(n));
 9133   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9134   ins_cost(VOLATILE_REF_COST);
 9135 
 9136   effect(KILL cr);
 9137 
 9138  format %{
 9139     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9140     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9141  %}
 9142 
 9143  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9144             aarch64_enc_cset_eq(res));
 9145 
 9146   ins_pipe(pipe_slow);
 9147 %}
 9148 
 9149 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9150 
 9151   predicate(needs_acquiring_load_exclusive(n));
 9152   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9153   ins_cost(VOLATILE_REF_COST);
 9154 
 9155   effect(KILL cr);
 9156 
 9157  format %{
 9158     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9159     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9160  %}
 9161 
 9162  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9163             aarch64_enc_cset_eq(res));
 9164 
 9165   ins_pipe(pipe_slow);
 9166 %}
 9167 
 9168 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9169 
 9170   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9171   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9172   ins_cost(VOLATILE_REF_COST);
 9173 
 9174   effect(KILL cr);
 9175 
 9176  format %{
 9177     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9178     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9179  %}
 9180 
 9181  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9182             aarch64_enc_cset_eq(res));
 9183 
 9184   ins_pipe(pipe_slow);
 9185 %}
 9186 
 9187 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9188 
 9189   predicate(needs_acquiring_load_exclusive(n));
 9190   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9191   ins_cost(VOLATILE_REF_COST);
 9192 
 9193   effect(KILL cr);
 9194 
 9195  format %{
 9196     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9197     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9198  %}
 9199 
 9200  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9201             aarch64_enc_cset_eq(res));
 9202 
 9203   ins_pipe(pipe_slow);
 9204 %}
 9205 
 9206 
 9207 // ---------------------------------------------------------------------
 9208 
 9209 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9210 
 9211 // Sundry CAS operations.  Note that release is always true,
 9212 // regardless of the memory ordering of the CAS.  This is because we
 9213 // need the volatile case to be sequentially consistent but there is
 9214 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9215 // can't check the type of memory ordering here, so we always emit a
 9216 // STLXR.
 9217 
 9218 // This section is generated from aarch64_ad_cas.m4
 9219 
 9220 
 9221 
 9222 // This pattern is generated automatically from cas.m4.
 9223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9224 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9225 
 9226   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9227   ins_cost(2 * VOLATILE_REF_COST);
 9228   effect(TEMP_DEF res, KILL cr);
 9229   format %{
 9230     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9231   %}
 9232   ins_encode %{
 9233     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9234                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9235                /*weak*/ false, $res$$Register);
 9236     __ sxtbw($res$$Register, $res$$Register);
 9237   %}
 9238   ins_pipe(pipe_slow);
 9239 %}
 9240 
 9241 // This pattern is generated automatically from cas.m4.
 9242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9243 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9244 
 9245   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9246   ins_cost(2 * VOLATILE_REF_COST);
 9247   effect(TEMP_DEF res, KILL cr);
 9248   format %{
 9249     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9250   %}
 9251   ins_encode %{
 9252     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9253                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9254                /*weak*/ false, $res$$Register);
 9255     __ sxthw($res$$Register, $res$$Register);
 9256   %}
 9257   ins_pipe(pipe_slow);
 9258 %}
 9259 
 9260 // This pattern is generated automatically from cas.m4.
 9261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9262 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9263 
 9264   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9265   ins_cost(2 * VOLATILE_REF_COST);
 9266   effect(TEMP_DEF res, KILL cr);
 9267   format %{
 9268     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9269   %}
 9270   ins_encode %{
 9271     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9272                Assembler::word, /*acquire*/ false, /*release*/ true,
 9273                /*weak*/ false, $res$$Register);
 9274   %}
 9275   ins_pipe(pipe_slow);
 9276 %}
 9277 
 9278 // This pattern is generated automatically from cas.m4.
 9279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9280 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9281 
 9282   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9283   ins_cost(2 * VOLATILE_REF_COST);
 9284   effect(TEMP_DEF res, KILL cr);
 9285   format %{
 9286     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9287   %}
 9288   ins_encode %{
 9289     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9290                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9291                /*weak*/ false, $res$$Register);
 9292   %}
 9293   ins_pipe(pipe_slow);
 9294 %}
 9295 
 9296 // This pattern is generated automatically from cas.m4.
 9297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9298 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9299 
 9300   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9301   ins_cost(2 * VOLATILE_REF_COST);
 9302   effect(TEMP_DEF res, KILL cr);
 9303   format %{
 9304     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9305   %}
 9306   ins_encode %{
 9307     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9308                Assembler::word, /*acquire*/ false, /*release*/ true,
 9309                /*weak*/ false, $res$$Register);
 9310   %}
 9311   ins_pipe(pipe_slow);
 9312 %}
 9313 
 9314 // This pattern is generated automatically from cas.m4.
 9315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9316 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9317   predicate(n->as_LoadStore()->barrier_data() == 0);
 9318   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9319   ins_cost(2 * VOLATILE_REF_COST);
 9320   effect(TEMP_DEF res, KILL cr);
 9321   format %{
 9322     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9323   %}
 9324   ins_encode %{
 9325     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9326                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9327                /*weak*/ false, $res$$Register);
 9328   %}
 9329   ins_pipe(pipe_slow);
 9330 %}
 9331 
 9332 // This pattern is generated automatically from cas.m4.
 9333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9334 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9335   predicate(needs_acquiring_load_exclusive(n));
 9336   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9337   ins_cost(VOLATILE_REF_COST);
 9338   effect(TEMP_DEF res, KILL cr);
 9339   format %{
 9340     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9341   %}
 9342   ins_encode %{
 9343     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9344                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9345                /*weak*/ false, $res$$Register);
 9346     __ sxtbw($res$$Register, $res$$Register);
 9347   %}
 9348   ins_pipe(pipe_slow);
 9349 %}
 9350 
 9351 // This pattern is generated automatically from cas.m4.
 9352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9353 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9354   predicate(needs_acquiring_load_exclusive(n));
 9355   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9356   ins_cost(VOLATILE_REF_COST);
 9357   effect(TEMP_DEF res, KILL cr);
 9358   format %{
 9359     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9360   %}
 9361   ins_encode %{
 9362     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9363                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9364                /*weak*/ false, $res$$Register);
 9365     __ sxthw($res$$Register, $res$$Register);
 9366   %}
 9367   ins_pipe(pipe_slow);
 9368 %}
 9369 
 9370 // This pattern is generated automatically from cas.m4.
 9371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9372 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9373   predicate(needs_acquiring_load_exclusive(n));
 9374   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9375   ins_cost(VOLATILE_REF_COST);
 9376   effect(TEMP_DEF res, KILL cr);
 9377   format %{
 9378     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9379   %}
 9380   ins_encode %{
 9381     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9382                Assembler::word, /*acquire*/ true, /*release*/ true,
 9383                /*weak*/ false, $res$$Register);
 9384   %}
 9385   ins_pipe(pipe_slow);
 9386 %}
 9387 
 9388 // This pattern is generated automatically from cas.m4.
 9389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9390 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9391   predicate(needs_acquiring_load_exclusive(n));
 9392   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9393   ins_cost(VOLATILE_REF_COST);
 9394   effect(TEMP_DEF res, KILL cr);
 9395   format %{
 9396     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9397   %}
 9398   ins_encode %{
 9399     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9400                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9401                /*weak*/ false, $res$$Register);
 9402   %}
 9403   ins_pipe(pipe_slow);
 9404 %}
 9405 
 9406 // This pattern is generated automatically from cas.m4.
 9407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9408 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9409   predicate(needs_acquiring_load_exclusive(n));
 9410   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9411   ins_cost(VOLATILE_REF_COST);
 9412   effect(TEMP_DEF res, KILL cr);
 9413   format %{
 9414     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9415   %}
 9416   ins_encode %{
 9417     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9418                Assembler::word, /*acquire*/ true, /*release*/ true,
 9419                /*weak*/ false, $res$$Register);
 9420   %}
 9421   ins_pipe(pipe_slow);
 9422 %}
 9423 
 9424 // This pattern is generated automatically from cas.m4.
 9425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9426 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9427   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9428   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9429   ins_cost(VOLATILE_REF_COST);
 9430   effect(TEMP_DEF res, KILL cr);
 9431   format %{
 9432     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9433   %}
 9434   ins_encode %{
 9435     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9436                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9437                /*weak*/ false, $res$$Register);
 9438   %}
 9439   ins_pipe(pipe_slow);
 9440 %}
 9441 
 9442 // This pattern is generated automatically from cas.m4.
 9443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9444 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9445 
 9446   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9447   ins_cost(2 * VOLATILE_REF_COST);
 9448   effect(KILL cr);
 9449   format %{
 9450     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9451     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9452   %}
 9453   ins_encode %{
 9454     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9455                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9456                /*weak*/ true, noreg);
 9457     __ csetw($res$$Register, Assembler::EQ);
 9458   %}
 9459   ins_pipe(pipe_slow);
 9460 %}
 9461 
 9462 // This pattern is generated automatically from cas.m4.
 9463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9464 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9465 
 9466   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9467   ins_cost(2 * VOLATILE_REF_COST);
 9468   effect(KILL cr);
 9469   format %{
 9470     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9471     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9472   %}
 9473   ins_encode %{
 9474     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9475                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9476                /*weak*/ true, noreg);
 9477     __ csetw($res$$Register, Assembler::EQ);
 9478   %}
 9479   ins_pipe(pipe_slow);
 9480 %}
 9481 
 9482 // This pattern is generated automatically from cas.m4.
 9483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9484 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9485 
 9486   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9487   ins_cost(2 * VOLATILE_REF_COST);
 9488   effect(KILL cr);
 9489   format %{
 9490     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9491     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9492   %}
 9493   ins_encode %{
 9494     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9495                Assembler::word, /*acquire*/ false, /*release*/ true,
 9496                /*weak*/ true, noreg);
 9497     __ csetw($res$$Register, Assembler::EQ);
 9498   %}
 9499   ins_pipe(pipe_slow);
 9500 %}
 9501 
 9502 // This pattern is generated automatically from cas.m4.
 9503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9504 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9505 
 9506   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9507   ins_cost(2 * VOLATILE_REF_COST);
 9508   effect(KILL cr);
 9509   format %{
 9510     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9511     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9512   %}
 9513   ins_encode %{
 9514     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9515                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9516                /*weak*/ true, noreg);
 9517     __ csetw($res$$Register, Assembler::EQ);
 9518   %}
 9519   ins_pipe(pipe_slow);
 9520 %}
 9521 
 9522 // This pattern is generated automatically from cas.m4.
 9523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9524 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9525 
 9526   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9527   ins_cost(2 * VOLATILE_REF_COST);
 9528   effect(KILL cr);
 9529   format %{
 9530     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9531     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9532   %}
 9533   ins_encode %{
 9534     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9535                Assembler::word, /*acquire*/ false, /*release*/ true,
 9536                /*weak*/ true, noreg);
 9537     __ csetw($res$$Register, Assembler::EQ);
 9538   %}
 9539   ins_pipe(pipe_slow);
 9540 %}
 9541 
 9542 // This pattern is generated automatically from cas.m4.
 9543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9544 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9545   predicate(n->as_LoadStore()->barrier_data() == 0);
 9546   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9547   ins_cost(2 * VOLATILE_REF_COST);
 9548   effect(KILL cr);
 9549   format %{
 9550     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9551     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9552   %}
 9553   ins_encode %{
 9554     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9555                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9556                /*weak*/ true, noreg);
 9557     __ csetw($res$$Register, Assembler::EQ);
 9558   %}
 9559   ins_pipe(pipe_slow);
 9560 %}
 9561 
 9562 // This pattern is generated automatically from cas.m4.
 9563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9564 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9565   predicate(needs_acquiring_load_exclusive(n));
 9566   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9567   ins_cost(VOLATILE_REF_COST);
 9568   effect(KILL cr);
 9569   format %{
 9570     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9571     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9572   %}
 9573   ins_encode %{
 9574     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9575                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9576                /*weak*/ true, noreg);
 9577     __ csetw($res$$Register, Assembler::EQ);
 9578   %}
 9579   ins_pipe(pipe_slow);
 9580 %}
 9581 
 9582 // This pattern is generated automatically from cas.m4.
 9583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9584 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9585   predicate(needs_acquiring_load_exclusive(n));
 9586   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9587   ins_cost(VOLATILE_REF_COST);
 9588   effect(KILL cr);
 9589   format %{
 9590     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9591     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9592   %}
 9593   ins_encode %{
 9594     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9595                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9596                /*weak*/ true, noreg);
 9597     __ csetw($res$$Register, Assembler::EQ);
 9598   %}
 9599   ins_pipe(pipe_slow);
 9600 %}
 9601 
 9602 // This pattern is generated automatically from cas.m4.
 9603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9604 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9605   predicate(needs_acquiring_load_exclusive(n));
 9606   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9607   ins_cost(VOLATILE_REF_COST);
 9608   effect(KILL cr);
 9609   format %{
 9610     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9611     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9612   %}
 9613   ins_encode %{
 9614     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9615                Assembler::word, /*acquire*/ true, /*release*/ true,
 9616                /*weak*/ true, noreg);
 9617     __ csetw($res$$Register, Assembler::EQ);
 9618   %}
 9619   ins_pipe(pipe_slow);
 9620 %}
 9621 
 9622 // This pattern is generated automatically from cas.m4.
 9623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9624 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9625   predicate(needs_acquiring_load_exclusive(n));
 9626   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9627   ins_cost(VOLATILE_REF_COST);
 9628   effect(KILL cr);
 9629   format %{
 9630     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9631     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9632   %}
 9633   ins_encode %{
 9634     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9635                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9636                /*weak*/ true, noreg);
 9637     __ csetw($res$$Register, Assembler::EQ);
 9638   %}
 9639   ins_pipe(pipe_slow);
 9640 %}
 9641 
 9642 // This pattern is generated automatically from cas.m4.
 9643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9644 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9645   predicate(needs_acquiring_load_exclusive(n));
 9646   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9647   ins_cost(VOLATILE_REF_COST);
 9648   effect(KILL cr);
 9649   format %{
 9650     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9651     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9652   %}
 9653   ins_encode %{
 9654     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9655                Assembler::word, /*acquire*/ true, /*release*/ true,
 9656                /*weak*/ true, noreg);
 9657     __ csetw($res$$Register, Assembler::EQ);
 9658   %}
 9659   ins_pipe(pipe_slow);
 9660 %}
 9661 
 9662 // This pattern is generated automatically from cas.m4.
 9663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9664 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9665   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9666   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9667   ins_cost(VOLATILE_REF_COST);
 9668   effect(KILL cr);
 9669   format %{
 9670     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9671     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9672   %}
 9673   ins_encode %{
 9674     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9675                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9676                /*weak*/ true, noreg);
 9677     __ csetw($res$$Register, Assembler::EQ);
 9678   %}
 9679   ins_pipe(pipe_slow);
 9680 %}
 9681 
 9682 // END This section of the file is automatically generated. Do not edit --------------
 9683 // ---------------------------------------------------------------------
 9684 
 9685 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9686   match(Set prev (GetAndSetI mem newv));
 9687   ins_cost(2 * VOLATILE_REF_COST);
 9688   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9689   ins_encode %{
 9690     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9691   %}
 9692   ins_pipe(pipe_serial);
 9693 %}
 9694 
 9695 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9696   match(Set prev (GetAndSetL mem newv));
 9697   ins_cost(2 * VOLATILE_REF_COST);
 9698   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9699   ins_encode %{
 9700     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9701   %}
 9702   ins_pipe(pipe_serial);
 9703 %}
 9704 
 9705 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9706   match(Set prev (GetAndSetN mem newv));
 9707   ins_cost(2 * VOLATILE_REF_COST);
 9708   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9709   ins_encode %{
 9710     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9711   %}
 9712   ins_pipe(pipe_serial);
 9713 %}
 9714 
 9715 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9716   predicate(n->as_LoadStore()->barrier_data() == 0);
 9717   match(Set prev (GetAndSetP mem newv));
 9718   ins_cost(2 * VOLATILE_REF_COST);
 9719   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9720   ins_encode %{
 9721     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9722   %}
 9723   ins_pipe(pipe_serial);
 9724 %}
 9725 
 9726 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9727   predicate(needs_acquiring_load_exclusive(n));
 9728   match(Set prev (GetAndSetI mem newv));
 9729   ins_cost(VOLATILE_REF_COST);
 9730   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9731   ins_encode %{
 9732     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9733   %}
 9734   ins_pipe(pipe_serial);
 9735 %}
 9736 
 9737 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9738   predicate(needs_acquiring_load_exclusive(n));
 9739   match(Set prev (GetAndSetL mem newv));
 9740   ins_cost(VOLATILE_REF_COST);
 9741   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9742   ins_encode %{
 9743     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9744   %}
 9745   ins_pipe(pipe_serial);
 9746 %}
 9747 
 9748 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9749   predicate(needs_acquiring_load_exclusive(n));
 9750   match(Set prev (GetAndSetN mem newv));
 9751   ins_cost(VOLATILE_REF_COST);
 9752   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9753   ins_encode %{
 9754     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9755   %}
 9756   ins_pipe(pipe_serial);
 9757 %}
 9758 
 9759 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9760   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9761   match(Set prev (GetAndSetP mem newv));
 9762   ins_cost(VOLATILE_REF_COST);
 9763   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9764   ins_encode %{
 9765     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9766   %}
 9767   ins_pipe(pipe_serial);
 9768 %}
 9769 
 9770 
 9771 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9772   match(Set newval (GetAndAddL mem incr));
 9773   ins_cost(2 * VOLATILE_REF_COST + 1);
 9774   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9775   ins_encode %{
 9776     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9777   %}
 9778   ins_pipe(pipe_serial);
 9779 %}
 9780 
 9781 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9782   predicate(n->as_LoadStore()->result_not_used());
 9783   match(Set dummy (GetAndAddL mem incr));
 9784   ins_cost(2 * VOLATILE_REF_COST);
 9785   format %{ "get_and_addL [$mem], $incr" %}
 9786   ins_encode %{
 9787     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9788   %}
 9789   ins_pipe(pipe_serial);
 9790 %}
 9791 
 9792 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9793   match(Set newval (GetAndAddL mem incr));
 9794   ins_cost(2 * VOLATILE_REF_COST + 1);
 9795   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9796   ins_encode %{
 9797     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9798   %}
 9799   ins_pipe(pipe_serial);
 9800 %}
 9801 
 9802 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9803   predicate(n->as_LoadStore()->result_not_used());
 9804   match(Set dummy (GetAndAddL mem incr));
 9805   ins_cost(2 * VOLATILE_REF_COST);
 9806   format %{ "get_and_addL [$mem], $incr" %}
 9807   ins_encode %{
 9808     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9809   %}
 9810   ins_pipe(pipe_serial);
 9811 %}
 9812 
 9813 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9814   match(Set newval (GetAndAddI mem incr));
 9815   ins_cost(2 * VOLATILE_REF_COST + 1);
 9816   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9817   ins_encode %{
 9818     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9819   %}
 9820   ins_pipe(pipe_serial);
 9821 %}
 9822 
 9823 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9824   predicate(n->as_LoadStore()->result_not_used());
 9825   match(Set dummy (GetAndAddI mem incr));
 9826   ins_cost(2 * VOLATILE_REF_COST);
 9827   format %{ "get_and_addI [$mem], $incr" %}
 9828   ins_encode %{
 9829     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9830   %}
 9831   ins_pipe(pipe_serial);
 9832 %}
 9833 
 9834 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9835   match(Set newval (GetAndAddI mem incr));
 9836   ins_cost(2 * VOLATILE_REF_COST + 1);
 9837   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9838   ins_encode %{
 9839     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9840   %}
 9841   ins_pipe(pipe_serial);
 9842 %}
 9843 
 9844 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9845   predicate(n->as_LoadStore()->result_not_used());
 9846   match(Set dummy (GetAndAddI mem incr));
 9847   ins_cost(2 * VOLATILE_REF_COST);
 9848   format %{ "get_and_addI [$mem], $incr" %}
 9849   ins_encode %{
 9850     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9851   %}
 9852   ins_pipe(pipe_serial);
 9853 %}
 9854 
 9855 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9856   predicate(needs_acquiring_load_exclusive(n));
 9857   match(Set newval (GetAndAddL mem incr));
 9858   ins_cost(VOLATILE_REF_COST + 1);
 9859   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9860   ins_encode %{
 9861     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9862   %}
 9863   ins_pipe(pipe_serial);
 9864 %}
 9865 
 9866 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9867   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9868   match(Set dummy (GetAndAddL mem incr));
 9869   ins_cost(VOLATILE_REF_COST);
 9870   format %{ "get_and_addL_acq [$mem], $incr" %}
 9871   ins_encode %{
 9872     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9873   %}
 9874   ins_pipe(pipe_serial);
 9875 %}
 9876 
 9877 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9878   predicate(needs_acquiring_load_exclusive(n));
 9879   match(Set newval (GetAndAddL mem incr));
 9880   ins_cost(VOLATILE_REF_COST + 1);
 9881   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9882   ins_encode %{
 9883     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9884   %}
 9885   ins_pipe(pipe_serial);
 9886 %}
 9887 
 9888 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9889   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9890   match(Set dummy (GetAndAddL mem incr));
 9891   ins_cost(VOLATILE_REF_COST);
 9892   format %{ "get_and_addL_acq [$mem], $incr" %}
 9893   ins_encode %{
 9894     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9895   %}
 9896   ins_pipe(pipe_serial);
 9897 %}
 9898 
 9899 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9900   predicate(needs_acquiring_load_exclusive(n));
 9901   match(Set newval (GetAndAddI mem incr));
 9902   ins_cost(VOLATILE_REF_COST + 1);
 9903   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9904   ins_encode %{
 9905     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9906   %}
 9907   ins_pipe(pipe_serial);
 9908 %}
 9909 
 9910 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9911   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9912   match(Set dummy (GetAndAddI mem incr));
 9913   ins_cost(VOLATILE_REF_COST);
 9914   format %{ "get_and_addI_acq [$mem], $incr" %}
 9915   ins_encode %{
 9916     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9917   %}
 9918   ins_pipe(pipe_serial);
 9919 %}
 9920 
 9921 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9922   predicate(needs_acquiring_load_exclusive(n));
 9923   match(Set newval (GetAndAddI mem incr));
 9924   ins_cost(VOLATILE_REF_COST + 1);
 9925   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9926   ins_encode %{
 9927     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9928   %}
 9929   ins_pipe(pipe_serial);
 9930 %}
 9931 
 9932 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9933   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9934   match(Set dummy (GetAndAddI mem incr));
 9935   ins_cost(VOLATILE_REF_COST);
 9936   format %{ "get_and_addI_acq [$mem], $incr" %}
 9937   ins_encode %{
 9938     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9939   %}
 9940   ins_pipe(pipe_serial);
 9941 %}
 9942 
 9943 // Manifest a CmpL result in an integer register.
 9944 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9945 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9946 %{
 9947   match(Set dst (CmpL3 src1 src2));
 9948   effect(KILL flags);
 9949 
 9950   ins_cost(INSN_COST * 6);
 9951   format %{
 9952       "cmp $src1, $src2"
 9953       "csetw $dst, ne"
 9954       "cnegw $dst, lt"
 9955   %}
 9956   // format %{ "CmpL3 $dst, $src1, $src2" %}
 9957   ins_encode %{
 9958     __ cmp($src1$$Register, $src2$$Register);
 9959     __ csetw($dst$$Register, Assembler::NE);
 9960     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9961   %}
 9962 
 9963   ins_pipe(pipe_class_default);
 9964 %}
 9965 
 9966 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9967 %{
 9968   match(Set dst (CmpL3 src1 src2));
 9969   effect(KILL flags);
 9970 
 9971   ins_cost(INSN_COST * 6);
 9972   format %{
 9973       "cmp $src1, $src2"
 9974       "csetw $dst, ne"
 9975       "cnegw $dst, lt"
 9976   %}
 9977   ins_encode %{
 9978     int32_t con = (int32_t)$src2$$constant;
 9979      if (con < 0) {
 9980       __ adds(zr, $src1$$Register, -con);
 9981     } else {
 9982       __ subs(zr, $src1$$Register, con);
 9983     }
 9984     __ csetw($dst$$Register, Assembler::NE);
 9985     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9986   %}
 9987 
 9988   ins_pipe(pipe_class_default);
 9989 %}
 9990 
 9991 // ============================================================================
 9992 // Conditional Move Instructions
 9993 
 9994 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9995 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9996 // define an op class which merged both inputs and use it to type the
 9997 // argument to a single rule. unfortunatelyt his fails because the
 9998 // opclass does not live up to the COND_INTER interface of its
 9999 // component operands. When the generic code tries to negate the
10000 // operand it ends up running the generci Machoper::negate method
10001 // which throws a ShouldNotHappen. So, we have to provide two flavours
10002 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10003 
10004 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10005   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10006 
10007   ins_cost(INSN_COST * 2);
10008   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10009 
10010   ins_encode %{
10011     __ cselw(as_Register($dst$$reg),
10012              as_Register($src2$$reg),
10013              as_Register($src1$$reg),
10014              (Assembler::Condition)$cmp$$cmpcode);
10015   %}
10016 
10017   ins_pipe(icond_reg_reg);
10018 %}
10019 
10020 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10021   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10022 
10023   ins_cost(INSN_COST * 2);
10024   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10025 
10026   ins_encode %{
10027     __ cselw(as_Register($dst$$reg),
10028              as_Register($src2$$reg),
10029              as_Register($src1$$reg),
10030              (Assembler::Condition)$cmp$$cmpcode);
10031   %}
10032 
10033   ins_pipe(icond_reg_reg);
10034 %}
10035 
10036 // special cases where one arg is zero
10037 
10038 // n.b. this is selected in preference to the rule above because it
10039 // avoids loading constant 0 into a source register
10040 
10041 // TODO
10042 // we ought only to be able to cull one of these variants as the ideal
10043 // transforms ought always to order the zero consistently (to left/right?)
10044 
10045 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10046   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10047 
10048   ins_cost(INSN_COST * 2);
10049   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10050 
10051   ins_encode %{
10052     __ cselw(as_Register($dst$$reg),
10053              as_Register($src$$reg),
10054              zr,
10055              (Assembler::Condition)$cmp$$cmpcode);
10056   %}
10057 
10058   ins_pipe(icond_reg);
10059 %}
10060 
10061 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10062   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10063 
10064   ins_cost(INSN_COST * 2);
10065   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10066 
10067   ins_encode %{
10068     __ cselw(as_Register($dst$$reg),
10069              as_Register($src$$reg),
10070              zr,
10071              (Assembler::Condition)$cmp$$cmpcode);
10072   %}
10073 
10074   ins_pipe(icond_reg);
10075 %}
10076 
10077 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10078   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10079 
10080   ins_cost(INSN_COST * 2);
10081   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10082 
10083   ins_encode %{
10084     __ cselw(as_Register($dst$$reg),
10085              zr,
10086              as_Register($src$$reg),
10087              (Assembler::Condition)$cmp$$cmpcode);
10088   %}
10089 
10090   ins_pipe(icond_reg);
10091 %}
10092 
10093 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10094   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10095 
10096   ins_cost(INSN_COST * 2);
10097   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10098 
10099   ins_encode %{
10100     __ cselw(as_Register($dst$$reg),
10101              zr,
10102              as_Register($src$$reg),
10103              (Assembler::Condition)$cmp$$cmpcode);
10104   %}
10105 
10106   ins_pipe(icond_reg);
10107 %}
10108 
10109 // special case for creating a boolean 0 or 1
10110 
10111 // n.b. this is selected in preference to the rule above because it
10112 // avoids loading constants 0 and 1 into a source register
10113 
10114 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10115   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10116 
10117   ins_cost(INSN_COST * 2);
10118   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10119 
10120   ins_encode %{
10121     // equivalently
10122     // cset(as_Register($dst$$reg),
10123     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10124     __ csincw(as_Register($dst$$reg),
10125              zr,
10126              zr,
10127              (Assembler::Condition)$cmp$$cmpcode);
10128   %}
10129 
10130   ins_pipe(icond_none);
10131 %}
10132 
10133 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10134   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10135 
10136   ins_cost(INSN_COST * 2);
10137   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10138 
10139   ins_encode %{
10140     // equivalently
10141     // cset(as_Register($dst$$reg),
10142     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10143     __ csincw(as_Register($dst$$reg),
10144              zr,
10145              zr,
10146              (Assembler::Condition)$cmp$$cmpcode);
10147   %}
10148 
10149   ins_pipe(icond_none);
10150 %}
10151 
10152 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10153   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10154 
10155   ins_cost(INSN_COST * 2);
10156   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10157 
10158   ins_encode %{
10159     __ csel(as_Register($dst$$reg),
10160             as_Register($src2$$reg),
10161             as_Register($src1$$reg),
10162             (Assembler::Condition)$cmp$$cmpcode);
10163   %}
10164 
10165   ins_pipe(icond_reg_reg);
10166 %}
10167 
10168 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10169   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10170 
10171   ins_cost(INSN_COST * 2);
10172   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10173 
10174   ins_encode %{
10175     __ csel(as_Register($dst$$reg),
10176             as_Register($src2$$reg),
10177             as_Register($src1$$reg),
10178             (Assembler::Condition)$cmp$$cmpcode);
10179   %}
10180 
10181   ins_pipe(icond_reg_reg);
10182 %}
10183 
10184 // special cases where one arg is zero
10185 
10186 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10187   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10188 
10189   ins_cost(INSN_COST * 2);
10190   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10191 
10192   ins_encode %{
10193     __ csel(as_Register($dst$$reg),
10194             zr,
10195             as_Register($src$$reg),
10196             (Assembler::Condition)$cmp$$cmpcode);
10197   %}
10198 
10199   ins_pipe(icond_reg);
10200 %}
10201 
10202 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10203   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10204 
10205   ins_cost(INSN_COST * 2);
10206   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10207 
10208   ins_encode %{
10209     __ csel(as_Register($dst$$reg),
10210             zr,
10211             as_Register($src$$reg),
10212             (Assembler::Condition)$cmp$$cmpcode);
10213   %}
10214 
10215   ins_pipe(icond_reg);
10216 %}
10217 
10218 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10219   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10220 
10221   ins_cost(INSN_COST * 2);
10222   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10223 
10224   ins_encode %{
10225     __ csel(as_Register($dst$$reg),
10226             as_Register($src$$reg),
10227             zr,
10228             (Assembler::Condition)$cmp$$cmpcode);
10229   %}
10230 
10231   ins_pipe(icond_reg);
10232 %}
10233 
10234 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10235   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10236 
10237   ins_cost(INSN_COST * 2);
10238   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10239 
10240   ins_encode %{
10241     __ csel(as_Register($dst$$reg),
10242             as_Register($src$$reg),
10243             zr,
10244             (Assembler::Condition)$cmp$$cmpcode);
10245   %}
10246 
10247   ins_pipe(icond_reg);
10248 %}
10249 
10250 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10251   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10252 
10253   ins_cost(INSN_COST * 2);
10254   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10255 
10256   ins_encode %{
10257     __ csel(as_Register($dst$$reg),
10258             as_Register($src2$$reg),
10259             as_Register($src1$$reg),
10260             (Assembler::Condition)$cmp$$cmpcode);
10261   %}
10262 
10263   ins_pipe(icond_reg_reg);
10264 %}
10265 
10266 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10267   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10268 
10269   ins_cost(INSN_COST * 2);
10270   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10271 
10272   ins_encode %{
10273     __ csel(as_Register($dst$$reg),
10274             as_Register($src2$$reg),
10275             as_Register($src1$$reg),
10276             (Assembler::Condition)$cmp$$cmpcode);
10277   %}
10278 
10279   ins_pipe(icond_reg_reg);
10280 %}
10281 
10282 // special cases where one arg is zero
10283 
10284 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10285   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10286 
10287   ins_cost(INSN_COST * 2);
10288   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10289 
10290   ins_encode %{
10291     __ csel(as_Register($dst$$reg),
10292             zr,
10293             as_Register($src$$reg),
10294             (Assembler::Condition)$cmp$$cmpcode);
10295   %}
10296 
10297   ins_pipe(icond_reg);
10298 %}
10299 
10300 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10301   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10302 
10303   ins_cost(INSN_COST * 2);
10304   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10305 
10306   ins_encode %{
10307     __ csel(as_Register($dst$$reg),
10308             zr,
10309             as_Register($src$$reg),
10310             (Assembler::Condition)$cmp$$cmpcode);
10311   %}
10312 
10313   ins_pipe(icond_reg);
10314 %}
10315 
10316 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10317   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10318 
10319   ins_cost(INSN_COST * 2);
10320   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10321 
10322   ins_encode %{
10323     __ csel(as_Register($dst$$reg),
10324             as_Register($src$$reg),
10325             zr,
10326             (Assembler::Condition)$cmp$$cmpcode);
10327   %}
10328 
10329   ins_pipe(icond_reg);
10330 %}
10331 
10332 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10333   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10334 
10335   ins_cost(INSN_COST * 2);
10336   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10337 
10338   ins_encode %{
10339     __ csel(as_Register($dst$$reg),
10340             as_Register($src$$reg),
10341             zr,
10342             (Assembler::Condition)$cmp$$cmpcode);
10343   %}
10344 
10345   ins_pipe(icond_reg);
10346 %}
10347 
10348 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10349   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10350 
10351   ins_cost(INSN_COST * 2);
10352   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10353 
10354   ins_encode %{
10355     __ cselw(as_Register($dst$$reg),
10356              as_Register($src2$$reg),
10357              as_Register($src1$$reg),
10358              (Assembler::Condition)$cmp$$cmpcode);
10359   %}
10360 
10361   ins_pipe(icond_reg_reg);
10362 %}
10363 
10364 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10365   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10366 
10367   ins_cost(INSN_COST * 2);
10368   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10369 
10370   ins_encode %{
10371     __ cselw(as_Register($dst$$reg),
10372              as_Register($src2$$reg),
10373              as_Register($src1$$reg),
10374              (Assembler::Condition)$cmp$$cmpcode);
10375   %}
10376 
10377   ins_pipe(icond_reg_reg);
10378 %}
10379 
10380 // special cases where one arg is zero
10381 
10382 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10383   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10384 
10385   ins_cost(INSN_COST * 2);
10386   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10387 
10388   ins_encode %{
10389     __ cselw(as_Register($dst$$reg),
10390              zr,
10391              as_Register($src$$reg),
10392              (Assembler::Condition)$cmp$$cmpcode);
10393   %}
10394 
10395   ins_pipe(icond_reg);
10396 %}
10397 
10398 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10399   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10400 
10401   ins_cost(INSN_COST * 2);
10402   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10403 
10404   ins_encode %{
10405     __ cselw(as_Register($dst$$reg),
10406              zr,
10407              as_Register($src$$reg),
10408              (Assembler::Condition)$cmp$$cmpcode);
10409   %}
10410 
10411   ins_pipe(icond_reg);
10412 %}
10413 
10414 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10415   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10416 
10417   ins_cost(INSN_COST * 2);
10418   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10419 
10420   ins_encode %{
10421     __ cselw(as_Register($dst$$reg),
10422              as_Register($src$$reg),
10423              zr,
10424              (Assembler::Condition)$cmp$$cmpcode);
10425   %}
10426 
10427   ins_pipe(icond_reg);
10428 %}
10429 
10430 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10431   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10432 
10433   ins_cost(INSN_COST * 2);
10434   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10435 
10436   ins_encode %{
10437     __ cselw(as_Register($dst$$reg),
10438              as_Register($src$$reg),
10439              zr,
10440              (Assembler::Condition)$cmp$$cmpcode);
10441   %}
10442 
10443   ins_pipe(icond_reg);
10444 %}
10445 
10446 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10447 %{
10448   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10449 
10450   ins_cost(INSN_COST * 3);
10451 
10452   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10453   ins_encode %{
10454     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10455     __ fcsels(as_FloatRegister($dst$$reg),
10456               as_FloatRegister($src2$$reg),
10457               as_FloatRegister($src1$$reg),
10458               cond);
10459   %}
10460 
10461   ins_pipe(fp_cond_reg_reg_s);
10462 %}
10463 
10464 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10465 %{
10466   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10467 
10468   ins_cost(INSN_COST * 3);
10469 
10470   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10471   ins_encode %{
10472     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10473     __ fcsels(as_FloatRegister($dst$$reg),
10474               as_FloatRegister($src2$$reg),
10475               as_FloatRegister($src1$$reg),
10476               cond);
10477   %}
10478 
10479   ins_pipe(fp_cond_reg_reg_s);
10480 %}
10481 
10482 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10483 %{
10484   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10485 
10486   ins_cost(INSN_COST * 3);
10487 
10488   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10489   ins_encode %{
10490     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10491     __ fcseld(as_FloatRegister($dst$$reg),
10492               as_FloatRegister($src2$$reg),
10493               as_FloatRegister($src1$$reg),
10494               cond);
10495   %}
10496 
10497   ins_pipe(fp_cond_reg_reg_d);
10498 %}
10499 
10500 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10501 %{
10502   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10503 
10504   ins_cost(INSN_COST * 3);
10505 
10506   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10507   ins_encode %{
10508     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10509     __ fcseld(as_FloatRegister($dst$$reg),
10510               as_FloatRegister($src2$$reg),
10511               as_FloatRegister($src1$$reg),
10512               cond);
10513   %}
10514 
10515   ins_pipe(fp_cond_reg_reg_d);
10516 %}
10517 
10518 // ============================================================================
10519 // Arithmetic Instructions
10520 //
10521 
10522 // Integer Addition
10523 
10524 // TODO
10525 // these currently employ operations which do not set CR and hence are
10526 // not flagged as killing CR but we would like to isolate the cases
10527 // where we want to set flags from those where we don't. need to work
10528 // out how to do that.
10529 
10530 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10531   match(Set dst (AddI src1 src2));
10532 
10533   ins_cost(INSN_COST);
10534   format %{ "addw  $dst, $src1, $src2" %}
10535 
10536   ins_encode %{
10537     __ addw(as_Register($dst$$reg),
10538             as_Register($src1$$reg),
10539             as_Register($src2$$reg));
10540   %}
10541 
10542   ins_pipe(ialu_reg_reg);
10543 %}
10544 
10545 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10546   match(Set dst (AddI src1 src2));
10547 
10548   ins_cost(INSN_COST);
10549   format %{ "addw $dst, $src1, $src2" %}
10550 
10551   // use opcode to indicate that this is an add not a sub
10552   opcode(0x0);
10553 
10554   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10555 
10556   ins_pipe(ialu_reg_imm);
10557 %}
10558 
10559 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10560   match(Set dst (AddI (ConvL2I src1) src2));
10561 
10562   ins_cost(INSN_COST);
10563   format %{ "addw $dst, $src1, $src2" %}
10564 
10565   // use opcode to indicate that this is an add not a sub
10566   opcode(0x0);
10567 
10568   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10569 
10570   ins_pipe(ialu_reg_imm);
10571 %}
10572 
10573 // Pointer Addition
10574 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10575   match(Set dst (AddP src1 src2));
10576 
10577   ins_cost(INSN_COST);
10578   format %{ "add $dst, $src1, $src2\t# ptr" %}
10579 
10580   ins_encode %{
10581     __ add(as_Register($dst$$reg),
10582            as_Register($src1$$reg),
10583            as_Register($src2$$reg));
10584   %}
10585 
10586   ins_pipe(ialu_reg_reg);
10587 %}
10588 
10589 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10590   match(Set dst (AddP src1 (ConvI2L src2)));
10591 
10592   ins_cost(1.9 * INSN_COST);
10593   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10594 
10595   ins_encode %{
10596     __ add(as_Register($dst$$reg),
10597            as_Register($src1$$reg),
10598            as_Register($src2$$reg), ext::sxtw);
10599   %}
10600 
10601   ins_pipe(ialu_reg_reg);
10602 %}
10603 
10604 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10605   match(Set dst (AddP src1 (LShiftL src2 scale)));
10606 
10607   ins_cost(1.9 * INSN_COST);
10608   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10609 
10610   ins_encode %{
10611     __ lea(as_Register($dst$$reg),
10612            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10613                    Address::lsl($scale$$constant)));
10614   %}
10615 
10616   ins_pipe(ialu_reg_reg_shift);
10617 %}
10618 
10619 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10620   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10621 
10622   ins_cost(1.9 * INSN_COST);
10623   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10624 
10625   ins_encode %{
10626     __ lea(as_Register($dst$$reg),
10627            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10628                    Address::sxtw($scale$$constant)));
10629   %}
10630 
10631   ins_pipe(ialu_reg_reg_shift);
10632 %}
10633 
10634 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10635   match(Set dst (LShiftL (ConvI2L src) scale));
10636 
10637   ins_cost(INSN_COST);
10638   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10639 
10640   ins_encode %{
10641     __ sbfiz(as_Register($dst$$reg),
10642           as_Register($src$$reg),
10643           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10644   %}
10645 
10646   ins_pipe(ialu_reg_shift);
10647 %}
10648 
10649 // Pointer Immediate Addition
10650 // n.b. this needs to be more expensive than using an indirect memory
10651 // operand
10652 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10653   match(Set dst (AddP src1 src2));
10654 
10655   ins_cost(INSN_COST);
10656   format %{ "add $dst, $src1, $src2\t# ptr" %}
10657 
10658   // use opcode to indicate that this is an add not a sub
10659   opcode(0x0);
10660 
10661   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10662 
10663   ins_pipe(ialu_reg_imm);
10664 %}
10665 
10666 // Long Addition
10667 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10668 
10669   match(Set dst (AddL src1 src2));
10670 
10671   ins_cost(INSN_COST);
10672   format %{ "add  $dst, $src1, $src2" %}
10673 
10674   ins_encode %{
10675     __ add(as_Register($dst$$reg),
10676            as_Register($src1$$reg),
10677            as_Register($src2$$reg));
10678   %}
10679 
10680   ins_pipe(ialu_reg_reg);
10681 %}
10682 
10683 // No constant pool entries requiredLong Immediate Addition.
10684 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10685   match(Set dst (AddL src1 src2));
10686 
10687   ins_cost(INSN_COST);
10688   format %{ "add $dst, $src1, $src2" %}
10689 
10690   // use opcode to indicate that this is an add not a sub
10691   opcode(0x0);
10692 
10693   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10694 
10695   ins_pipe(ialu_reg_imm);
10696 %}
10697 
10698 // Integer Subtraction
10699 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10700   match(Set dst (SubI src1 src2));
10701 
10702   ins_cost(INSN_COST);
10703   format %{ "subw  $dst, $src1, $src2" %}
10704 
10705   ins_encode %{
10706     __ subw(as_Register($dst$$reg),
10707             as_Register($src1$$reg),
10708             as_Register($src2$$reg));
10709   %}
10710 
10711   ins_pipe(ialu_reg_reg);
10712 %}
10713 
10714 // Immediate Subtraction
10715 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10716   match(Set dst (SubI src1 src2));
10717 
10718   ins_cost(INSN_COST);
10719   format %{ "subw $dst, $src1, $src2" %}
10720 
10721   // use opcode to indicate that this is a sub not an add
10722   opcode(0x1);
10723 
10724   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10725 
10726   ins_pipe(ialu_reg_imm);
10727 %}
10728 
10729 // Long Subtraction
10730 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10731 
10732   match(Set dst (SubL src1 src2));
10733 
10734   ins_cost(INSN_COST);
10735   format %{ "sub  $dst, $src1, $src2" %}
10736 
10737   ins_encode %{
10738     __ sub(as_Register($dst$$reg),
10739            as_Register($src1$$reg),
10740            as_Register($src2$$reg));
10741   %}
10742 
10743   ins_pipe(ialu_reg_reg);
10744 %}
10745 
10746 // No constant pool entries requiredLong Immediate Subtraction.
10747 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10748   match(Set dst (SubL src1 src2));
10749 
10750   ins_cost(INSN_COST);
10751   format %{ "sub$dst, $src1, $src2" %}
10752 
10753   // use opcode to indicate that this is a sub not an add
10754   opcode(0x1);
10755 
10756   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10757 
10758   ins_pipe(ialu_reg_imm);
10759 %}
10760 
10761 // Integer Negation (special case for sub)
10762 
10763 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10764   match(Set dst (SubI zero src));
10765 
10766   ins_cost(INSN_COST);
10767   format %{ "negw $dst, $src\t# int" %}
10768 
10769   ins_encode %{
10770     __ negw(as_Register($dst$$reg),
10771             as_Register($src$$reg));
10772   %}
10773 
10774   ins_pipe(ialu_reg);
10775 %}
10776 
10777 // Long Negation
10778 
10779 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10780   match(Set dst (SubL zero src));
10781 
10782   ins_cost(INSN_COST);
10783   format %{ "neg $dst, $src\t# long" %}
10784 
10785   ins_encode %{
10786     __ neg(as_Register($dst$$reg),
10787            as_Register($src$$reg));
10788   %}
10789 
10790   ins_pipe(ialu_reg);
10791 %}
10792 
10793 // Integer Multiply
10794 
10795 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10796   match(Set dst (MulI src1 src2));
10797 
10798   ins_cost(INSN_COST * 3);
10799   format %{ "mulw  $dst, $src1, $src2" %}
10800 
10801   ins_encode %{
10802     __ mulw(as_Register($dst$$reg),
10803             as_Register($src1$$reg),
10804             as_Register($src2$$reg));
10805   %}
10806 
10807   ins_pipe(imul_reg_reg);
10808 %}
10809 
10810 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10811   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10812 
10813   ins_cost(INSN_COST * 3);
10814   format %{ "smull  $dst, $src1, $src2" %}
10815 
10816   ins_encode %{
10817     __ smull(as_Register($dst$$reg),
10818              as_Register($src1$$reg),
10819              as_Register($src2$$reg));
10820   %}
10821 
10822   ins_pipe(imul_reg_reg);
10823 %}
10824 
10825 // Long Multiply
10826 
10827 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10828   match(Set dst (MulL src1 src2));
10829 
10830   ins_cost(INSN_COST * 5);
10831   format %{ "mul  $dst, $src1, $src2" %}
10832 
10833   ins_encode %{
10834     __ mul(as_Register($dst$$reg),
10835            as_Register($src1$$reg),
10836            as_Register($src2$$reg));
10837   %}
10838 
10839   ins_pipe(lmul_reg_reg);
10840 %}
10841 
10842 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10843 %{
10844   match(Set dst (MulHiL src1 src2));
10845 
10846   ins_cost(INSN_COST * 7);
10847   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10848 
10849   ins_encode %{
10850     __ smulh(as_Register($dst$$reg),
10851              as_Register($src1$$reg),
10852              as_Register($src2$$reg));
10853   %}
10854 
10855   ins_pipe(lmul_reg_reg);
10856 %}
10857 
10858 // Combined Integer Multiply & Add/Sub
10859 
10860 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10861   match(Set dst (AddI src3 (MulI src1 src2)));
10862 
10863   ins_cost(INSN_COST * 3);
10864   format %{ "madd  $dst, $src1, $src2, $src3" %}
10865 
10866   ins_encode %{
10867     __ maddw(as_Register($dst$$reg),
10868              as_Register($src1$$reg),
10869              as_Register($src2$$reg),
10870              as_Register($src3$$reg));
10871   %}
10872 
10873   ins_pipe(imac_reg_reg);
10874 %}
10875 
10876 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10877   match(Set dst (SubI src3 (MulI src1 src2)));
10878 
10879   ins_cost(INSN_COST * 3);
10880   format %{ "msub  $dst, $src1, $src2, $src3" %}
10881 
10882   ins_encode %{
10883     __ msubw(as_Register($dst$$reg),
10884              as_Register($src1$$reg),
10885              as_Register($src2$$reg),
10886              as_Register($src3$$reg));
10887   %}
10888 
10889   ins_pipe(imac_reg_reg);
10890 %}
10891 
10892 // Combined Integer Multiply & Neg
10893 
10894 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10895   match(Set dst (MulI (SubI zero src1) src2));
10896 
10897   ins_cost(INSN_COST * 3);
10898   format %{ "mneg  $dst, $src1, $src2" %}
10899 
10900   ins_encode %{
10901     __ mnegw(as_Register($dst$$reg),
10902              as_Register($src1$$reg),
10903              as_Register($src2$$reg));
10904   %}
10905 
10906   ins_pipe(imac_reg_reg);
10907 %}
10908 
10909 // Combined Long Multiply & Add/Sub
10910 
10911 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10912   match(Set dst (AddL src3 (MulL src1 src2)));
10913 
10914   ins_cost(INSN_COST * 5);
10915   format %{ "madd  $dst, $src1, $src2, $src3" %}
10916 
10917   ins_encode %{
10918     __ madd(as_Register($dst$$reg),
10919             as_Register($src1$$reg),
10920             as_Register($src2$$reg),
10921             as_Register($src3$$reg));
10922   %}
10923 
10924   ins_pipe(lmac_reg_reg);
10925 %}
10926 
10927 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10928   match(Set dst (SubL src3 (MulL src1 src2)));
10929 
10930   ins_cost(INSN_COST * 5);
10931   format %{ "msub  $dst, $src1, $src2, $src3" %}
10932 
10933   ins_encode %{
10934     __ msub(as_Register($dst$$reg),
10935             as_Register($src1$$reg),
10936             as_Register($src2$$reg),
10937             as_Register($src3$$reg));
10938   %}
10939 
10940   ins_pipe(lmac_reg_reg);
10941 %}
10942 
10943 // Combined Long Multiply & Neg
10944 
10945 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10946   match(Set dst (MulL (SubL zero src1) src2));
10947 
10948   ins_cost(INSN_COST * 5);
10949   format %{ "mneg  $dst, $src1, $src2" %}
10950 
10951   ins_encode %{
10952     __ mneg(as_Register($dst$$reg),
10953             as_Register($src1$$reg),
10954             as_Register($src2$$reg));
10955   %}
10956 
10957   ins_pipe(lmac_reg_reg);
10958 %}
10959 
10960 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10961 
10962 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10963   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10964 
10965   ins_cost(INSN_COST * 3);
10966   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10967 
10968   ins_encode %{
10969     __ smaddl(as_Register($dst$$reg),
10970               as_Register($src1$$reg),
10971               as_Register($src2$$reg),
10972               as_Register($src3$$reg));
10973   %}
10974 
10975   ins_pipe(imac_reg_reg);
10976 %}
10977 
10978 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10979   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10980 
10981   ins_cost(INSN_COST * 3);
10982   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10983 
10984   ins_encode %{
10985     __ smsubl(as_Register($dst$$reg),
10986               as_Register($src1$$reg),
10987               as_Register($src2$$reg),
10988               as_Register($src3$$reg));
10989   %}
10990 
10991   ins_pipe(imac_reg_reg);
10992 %}
10993 
10994 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10995   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10996 
10997   ins_cost(INSN_COST * 3);
10998   format %{ "smnegl  $dst, $src1, $src2" %}
10999 
11000   ins_encode %{
11001     __ smnegl(as_Register($dst$$reg),
11002               as_Register($src1$$reg),
11003               as_Register($src2$$reg));
11004   %}
11005 
11006   ins_pipe(imac_reg_reg);
11007 %}
11008 
11009 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11010 
11011 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11012   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11013 
11014   ins_cost(INSN_COST * 5);
11015   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11016             "maddw $dst, $src3, $src4, rscratch1" %}
11017 
11018   ins_encode %{
11019     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11020     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11021 
11022   ins_pipe(imac_reg_reg);
11023 %}
11024 
11025 // Integer Divide
11026 
11027 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11028   match(Set dst (DivI src1 src2));
11029 
11030   ins_cost(INSN_COST * 19);
11031   format %{ "sdivw  $dst, $src1, $src2" %}
11032 
11033   ins_encode(aarch64_enc_divw(dst, src1, src2));
11034   ins_pipe(idiv_reg_reg);
11035 %}
11036 
11037 // Long Divide
11038 
11039 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11040   match(Set dst (DivL src1 src2));
11041 
11042   ins_cost(INSN_COST * 35);
11043   format %{ "sdiv   $dst, $src1, $src2" %}
11044 
11045   ins_encode(aarch64_enc_div(dst, src1, src2));
11046   ins_pipe(ldiv_reg_reg);
11047 %}
11048 
11049 // Integer Remainder
11050 
11051 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11052   match(Set dst (ModI src1 src2));
11053 
11054   ins_cost(INSN_COST * 22);
11055   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11056             "msubw($dst, rscratch1, $src2, $src1" %}
11057 
11058   ins_encode(aarch64_enc_modw(dst, src1, src2));
11059   ins_pipe(idiv_reg_reg);
11060 %}
11061 
11062 // Long Remainder
11063 
11064 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11065   match(Set dst (ModL src1 src2));
11066 
11067   ins_cost(INSN_COST * 38);
11068   format %{ "sdiv   rscratch1, $src1, $src2\n"
11069             "msub($dst, rscratch1, $src2, $src1" %}
11070 
11071   ins_encode(aarch64_enc_mod(dst, src1, src2));
11072   ins_pipe(ldiv_reg_reg);
11073 %}
11074 
11075 // Integer Shifts
11076 
11077 // Shift Left Register
11078 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11079   match(Set dst (LShiftI src1 src2));
11080 
11081   ins_cost(INSN_COST * 2);
11082   format %{ "lslvw  $dst, $src1, $src2" %}
11083 
11084   ins_encode %{
11085     __ lslvw(as_Register($dst$$reg),
11086              as_Register($src1$$reg),
11087              as_Register($src2$$reg));
11088   %}
11089 
11090   ins_pipe(ialu_reg_reg_vshift);
11091 %}
11092 
11093 // Shift Left Immediate
11094 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11095   match(Set dst (LShiftI src1 src2));
11096 
11097   ins_cost(INSN_COST);
11098   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11099 
11100   ins_encode %{
11101     __ lslw(as_Register($dst$$reg),
11102             as_Register($src1$$reg),
11103             $src2$$constant & 0x1f);
11104   %}
11105 
11106   ins_pipe(ialu_reg_shift);
11107 %}
11108 
11109 // Shift Right Logical Register
11110 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11111   match(Set dst (URShiftI src1 src2));
11112 
11113   ins_cost(INSN_COST * 2);
11114   format %{ "lsrvw  $dst, $src1, $src2" %}
11115 
11116   ins_encode %{
11117     __ lsrvw(as_Register($dst$$reg),
11118              as_Register($src1$$reg),
11119              as_Register($src2$$reg));
11120   %}
11121 
11122   ins_pipe(ialu_reg_reg_vshift);
11123 %}
11124 
11125 // Shift Right Logical Immediate
11126 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11127   match(Set dst (URShiftI src1 src2));
11128 
11129   ins_cost(INSN_COST);
11130   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11131 
11132   ins_encode %{
11133     __ lsrw(as_Register($dst$$reg),
11134             as_Register($src1$$reg),
11135             $src2$$constant & 0x1f);
11136   %}
11137 
11138   ins_pipe(ialu_reg_shift);
11139 %}
11140 
11141 // Shift Right Arithmetic Register
11142 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11143   match(Set dst (RShiftI src1 src2));
11144 
11145   ins_cost(INSN_COST * 2);
11146   format %{ "asrvw  $dst, $src1, $src2" %}
11147 
11148   ins_encode %{
11149     __ asrvw(as_Register($dst$$reg),
11150              as_Register($src1$$reg),
11151              as_Register($src2$$reg));
11152   %}
11153 
11154   ins_pipe(ialu_reg_reg_vshift);
11155 %}
11156 
11157 // Shift Right Arithmetic Immediate
11158 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11159   match(Set dst (RShiftI src1 src2));
11160 
11161   ins_cost(INSN_COST);
11162   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11163 
11164   ins_encode %{
11165     __ asrw(as_Register($dst$$reg),
11166             as_Register($src1$$reg),
11167             $src2$$constant & 0x1f);
11168   %}
11169 
11170   ins_pipe(ialu_reg_shift);
11171 %}
11172 
11173 // Combined Int Mask and Right Shift (using UBFM)
11174 // TODO
11175 
11176 // Long Shifts
11177 
11178 // Shift Left Register
11179 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11180   match(Set dst (LShiftL src1 src2));
11181 
11182   ins_cost(INSN_COST * 2);
11183   format %{ "lslv  $dst, $src1, $src2" %}
11184 
11185   ins_encode %{
11186     __ lslv(as_Register($dst$$reg),
11187             as_Register($src1$$reg),
11188             as_Register($src2$$reg));
11189   %}
11190 
11191   ins_pipe(ialu_reg_reg_vshift);
11192 %}
11193 
11194 // Shift Left Immediate
11195 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11196   match(Set dst (LShiftL src1 src2));
11197 
11198   ins_cost(INSN_COST);
11199   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11200 
11201   ins_encode %{
11202     __ lsl(as_Register($dst$$reg),
11203             as_Register($src1$$reg),
11204             $src2$$constant & 0x3f);
11205   %}
11206 
11207   ins_pipe(ialu_reg_shift);
11208 %}
11209 
11210 // Shift Right Logical Register
11211 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11212   match(Set dst (URShiftL src1 src2));
11213 
11214   ins_cost(INSN_COST * 2);
11215   format %{ "lsrv  $dst, $src1, $src2" %}
11216 
11217   ins_encode %{
11218     __ lsrv(as_Register($dst$$reg),
11219             as_Register($src1$$reg),
11220             as_Register($src2$$reg));
11221   %}
11222 
11223   ins_pipe(ialu_reg_reg_vshift);
11224 %}
11225 
11226 // Shift Right Logical Immediate
11227 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11228   match(Set dst (URShiftL src1 src2));
11229 
11230   ins_cost(INSN_COST);
11231   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11232 
11233   ins_encode %{
11234     __ lsr(as_Register($dst$$reg),
11235            as_Register($src1$$reg),
11236            $src2$$constant & 0x3f);
11237   %}
11238 
11239   ins_pipe(ialu_reg_shift);
11240 %}
11241 
11242 // A special-case pattern for card table stores.
11243 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11244   match(Set dst (URShiftL (CastP2X src1) src2));
11245 
11246   ins_cost(INSN_COST);
11247   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11248 
11249   ins_encode %{
11250     __ lsr(as_Register($dst$$reg),
11251            as_Register($src1$$reg),
11252            $src2$$constant & 0x3f);
11253   %}
11254 
11255   ins_pipe(ialu_reg_shift);
11256 %}
11257 
11258 // Shift Right Arithmetic Register
11259 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11260   match(Set dst (RShiftL src1 src2));
11261 
11262   ins_cost(INSN_COST * 2);
11263   format %{ "asrv  $dst, $src1, $src2" %}
11264 
11265   ins_encode %{
11266     __ asrv(as_Register($dst$$reg),
11267             as_Register($src1$$reg),
11268             as_Register($src2$$reg));
11269   %}
11270 
11271   ins_pipe(ialu_reg_reg_vshift);
11272 %}
11273 
11274 // Shift Right Arithmetic Immediate
11275 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11276   match(Set dst (RShiftL src1 src2));
11277 
11278   ins_cost(INSN_COST);
11279   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11280 
11281   ins_encode %{
11282     __ asr(as_Register($dst$$reg),
11283            as_Register($src1$$reg),
11284            $src2$$constant & 0x3f);
11285   %}
11286 
11287   ins_pipe(ialu_reg_shift);
11288 %}
11289 
11290 // BEGIN This section of the file is automatically generated. Do not edit --------------
11291 // This section is generated from aarch64_ad.m4
11292 
11293 
11294 // This pattern is automatically generated from aarch64_ad.m4
11295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11296 instruct regL_not_reg(iRegLNoSp dst,
11297                          iRegL src1, immL_M1 m1,
11298                          rFlagsReg cr) %{
11299   match(Set dst (XorL src1 m1));
11300   ins_cost(INSN_COST);
11301   format %{ "eon  $dst, $src1, zr" %}
11302 
11303   ins_encode %{
11304     __ eon(as_Register($dst$$reg),
11305               as_Register($src1$$reg),
11306               zr,
11307               Assembler::LSL, 0);
11308   %}
11309 
11310   ins_pipe(ialu_reg);
11311 %}
11312 
11313 // This pattern is automatically generated from aarch64_ad.m4
11314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11315 instruct regI_not_reg(iRegINoSp dst,
11316                          iRegIorL2I src1, immI_M1 m1,
11317                          rFlagsReg cr) %{
11318   match(Set dst (XorI src1 m1));
11319   ins_cost(INSN_COST);
11320   format %{ "eonw  $dst, $src1, zr" %}
11321 
11322   ins_encode %{
11323     __ eonw(as_Register($dst$$reg),
11324               as_Register($src1$$reg),
11325               zr,
11326               Assembler::LSL, 0);
11327   %}
11328 
11329   ins_pipe(ialu_reg);
11330 %}
11331 
11332 // This pattern is automatically generated from aarch64_ad.m4
11333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11334 instruct AndI_reg_not_reg(iRegINoSp dst,
11335                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11336   match(Set dst (AndI src1 (XorI src2 m1)));
11337   ins_cost(INSN_COST);
11338   format %{ "bicw  $dst, $src1, $src2" %}
11339 
11340   ins_encode %{
11341     __ bicw(as_Register($dst$$reg),
11342               as_Register($src1$$reg),
11343               as_Register($src2$$reg),
11344               Assembler::LSL, 0);
11345   %}
11346 
11347   ins_pipe(ialu_reg_reg);
11348 %}
11349 
11350 // This pattern is automatically generated from aarch64_ad.m4
11351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11352 instruct AndL_reg_not_reg(iRegLNoSp dst,
11353                          iRegL src1, iRegL src2, immL_M1 m1) %{
11354   match(Set dst (AndL src1 (XorL src2 m1)));
11355   ins_cost(INSN_COST);
11356   format %{ "bic  $dst, $src1, $src2" %}
11357 
11358   ins_encode %{
11359     __ bic(as_Register($dst$$reg),
11360               as_Register($src1$$reg),
11361               as_Register($src2$$reg),
11362               Assembler::LSL, 0);
11363   %}
11364 
11365   ins_pipe(ialu_reg_reg);
11366 %}
11367 
11368 // This pattern is automatically generated from aarch64_ad.m4
11369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11370 instruct OrI_reg_not_reg(iRegINoSp dst,
11371                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11372   match(Set dst (OrI src1 (XorI src2 m1)));
11373   ins_cost(INSN_COST);
11374   format %{ "ornw  $dst, $src1, $src2" %}
11375 
11376   ins_encode %{
11377     __ ornw(as_Register($dst$$reg),
11378               as_Register($src1$$reg),
11379               as_Register($src2$$reg),
11380               Assembler::LSL, 0);
11381   %}
11382 
11383   ins_pipe(ialu_reg_reg);
11384 %}
11385 
11386 // This pattern is automatically generated from aarch64_ad.m4
11387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11388 instruct OrL_reg_not_reg(iRegLNoSp dst,
11389                          iRegL src1, iRegL src2, immL_M1 m1) %{
11390   match(Set dst (OrL src1 (XorL src2 m1)));
11391   ins_cost(INSN_COST);
11392   format %{ "orn  $dst, $src1, $src2" %}
11393 
11394   ins_encode %{
11395     __ orn(as_Register($dst$$reg),
11396               as_Register($src1$$reg),
11397               as_Register($src2$$reg),
11398               Assembler::LSL, 0);
11399   %}
11400 
11401   ins_pipe(ialu_reg_reg);
11402 %}
11403 
11404 // This pattern is automatically generated from aarch64_ad.m4
11405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11406 instruct XorI_reg_not_reg(iRegINoSp dst,
11407                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11408   match(Set dst (XorI m1 (XorI src2 src1)));
11409   ins_cost(INSN_COST);
11410   format %{ "eonw  $dst, $src1, $src2" %}
11411 
11412   ins_encode %{
11413     __ eonw(as_Register($dst$$reg),
11414               as_Register($src1$$reg),
11415               as_Register($src2$$reg),
11416               Assembler::LSL, 0);
11417   %}
11418 
11419   ins_pipe(ialu_reg_reg);
11420 %}
11421 
11422 // This pattern is automatically generated from aarch64_ad.m4
11423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11424 instruct XorL_reg_not_reg(iRegLNoSp dst,
11425                          iRegL src1, iRegL src2, immL_M1 m1) %{
11426   match(Set dst (XorL m1 (XorL src2 src1)));
11427   ins_cost(INSN_COST);
11428   format %{ "eon  $dst, $src1, $src2" %}
11429 
11430   ins_encode %{
11431     __ eon(as_Register($dst$$reg),
11432               as_Register($src1$$reg),
11433               as_Register($src2$$reg),
11434               Assembler::LSL, 0);
11435   %}
11436 
11437   ins_pipe(ialu_reg_reg);
11438 %}
11439 
11440 // This pattern is automatically generated from aarch64_ad.m4
11441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11442 // val & (-1 ^ (val >>> shift)) ==> bicw
11443 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11444                          iRegIorL2I src1, iRegIorL2I src2,
11445                          immI src3, immI_M1 src4) %{
11446   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11447   ins_cost(1.9 * INSN_COST);
11448   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11449 
11450   ins_encode %{
11451     __ bicw(as_Register($dst$$reg),
11452               as_Register($src1$$reg),
11453               as_Register($src2$$reg),
11454               Assembler::LSR,
11455               $src3$$constant & 0x1f);
11456   %}
11457 
11458   ins_pipe(ialu_reg_reg_shift);
11459 %}
11460 
11461 // This pattern is automatically generated from aarch64_ad.m4
11462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11463 // val & (-1 ^ (val >>> shift)) ==> bic
11464 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11465                          iRegL src1, iRegL src2,
11466                          immI src3, immL_M1 src4) %{
11467   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11468   ins_cost(1.9 * INSN_COST);
11469   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11470 
11471   ins_encode %{
11472     __ bic(as_Register($dst$$reg),
11473               as_Register($src1$$reg),
11474               as_Register($src2$$reg),
11475               Assembler::LSR,
11476               $src3$$constant & 0x3f);
11477   %}
11478 
11479   ins_pipe(ialu_reg_reg_shift);
11480 %}
11481 
11482 // This pattern is automatically generated from aarch64_ad.m4
11483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11484 // val & (-1 ^ (val >> shift)) ==> bicw
11485 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11486                          iRegIorL2I src1, iRegIorL2I src2,
11487                          immI src3, immI_M1 src4) %{
11488   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11489   ins_cost(1.9 * INSN_COST);
11490   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11491 
11492   ins_encode %{
11493     __ bicw(as_Register($dst$$reg),
11494               as_Register($src1$$reg),
11495               as_Register($src2$$reg),
11496               Assembler::ASR,
11497               $src3$$constant & 0x1f);
11498   %}
11499 
11500   ins_pipe(ialu_reg_reg_shift);
11501 %}
11502 
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505 // val & (-1 ^ (val >> shift)) ==> bic
11506 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11507                          iRegL src1, iRegL src2,
11508                          immI src3, immL_M1 src4) %{
11509   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11510   ins_cost(1.9 * INSN_COST);
11511   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11512 
11513   ins_encode %{
11514     __ bic(as_Register($dst$$reg),
11515               as_Register($src1$$reg),
11516               as_Register($src2$$reg),
11517               Assembler::ASR,
11518               $src3$$constant & 0x3f);
11519   %}
11520 
11521   ins_pipe(ialu_reg_reg_shift);
11522 %}
11523 
11524 // This pattern is automatically generated from aarch64_ad.m4
11525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11526 // val & (-1 ^ (val ror shift)) ==> bicw
11527 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11528                          iRegIorL2I src1, iRegIorL2I src2,
11529                          immI src3, immI_M1 src4) %{
11530   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11531   ins_cost(1.9 * INSN_COST);
11532   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11533 
11534   ins_encode %{
11535     __ bicw(as_Register($dst$$reg),
11536               as_Register($src1$$reg),
11537               as_Register($src2$$reg),
11538               Assembler::ROR,
11539               $src3$$constant & 0x1f);
11540   %}
11541 
11542   ins_pipe(ialu_reg_reg_shift);
11543 %}
11544 
11545 // This pattern is automatically generated from aarch64_ad.m4
11546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11547 // val & (-1 ^ (val ror shift)) ==> bic
11548 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11549                          iRegL src1, iRegL src2,
11550                          immI src3, immL_M1 src4) %{
11551   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11552   ins_cost(1.9 * INSN_COST);
11553   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11554 
11555   ins_encode %{
11556     __ bic(as_Register($dst$$reg),
11557               as_Register($src1$$reg),
11558               as_Register($src2$$reg),
11559               Assembler::ROR,
11560               $src3$$constant & 0x3f);
11561   %}
11562 
11563   ins_pipe(ialu_reg_reg_shift);
11564 %}
11565 
11566 // This pattern is automatically generated from aarch64_ad.m4
11567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11568 // val & (-1 ^ (val << shift)) ==> bicw
11569 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11570                          iRegIorL2I src1, iRegIorL2I src2,
11571                          immI src3, immI_M1 src4) %{
11572   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11573   ins_cost(1.9 * INSN_COST);
11574   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11575 
11576   ins_encode %{
11577     __ bicw(as_Register($dst$$reg),
11578               as_Register($src1$$reg),
11579               as_Register($src2$$reg),
11580               Assembler::LSL,
11581               $src3$$constant & 0x1f);
11582   %}
11583 
11584   ins_pipe(ialu_reg_reg_shift);
11585 %}
11586 
11587 // This pattern is automatically generated from aarch64_ad.m4
11588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11589 // val & (-1 ^ (val << shift)) ==> bic
11590 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11591                          iRegL src1, iRegL src2,
11592                          immI src3, immL_M1 src4) %{
11593   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11594   ins_cost(1.9 * INSN_COST);
11595   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11596 
11597   ins_encode %{
11598     __ bic(as_Register($dst$$reg),
11599               as_Register($src1$$reg),
11600               as_Register($src2$$reg),
11601               Assembler::LSL,
11602               $src3$$constant & 0x3f);
11603   %}
11604 
11605   ins_pipe(ialu_reg_reg_shift);
11606 %}
11607 
11608 // This pattern is automatically generated from aarch64_ad.m4
11609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11610 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11611 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11612                          iRegIorL2I src1, iRegIorL2I src2,
11613                          immI src3, immI_M1 src4) %{
11614   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11615   ins_cost(1.9 * INSN_COST);
11616   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11617 
11618   ins_encode %{
11619     __ eonw(as_Register($dst$$reg),
11620               as_Register($src1$$reg),
11621               as_Register($src2$$reg),
11622               Assembler::LSR,
11623               $src3$$constant & 0x1f);
11624   %}
11625 
11626   ins_pipe(ialu_reg_reg_shift);
11627 %}
11628 
11629 // This pattern is automatically generated from aarch64_ad.m4
11630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11631 // val ^ (-1 ^ (val >>> shift)) ==> eon
11632 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11633                          iRegL src1, iRegL src2,
11634                          immI src3, immL_M1 src4) %{
11635   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11636   ins_cost(1.9 * INSN_COST);
11637   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11638 
11639   ins_encode %{
11640     __ eon(as_Register($dst$$reg),
11641               as_Register($src1$$reg),
11642               as_Register($src2$$reg),
11643               Assembler::LSR,
11644               $src3$$constant & 0x3f);
11645   %}
11646 
11647   ins_pipe(ialu_reg_reg_shift);
11648 %}
11649 
11650 // This pattern is automatically generated from aarch64_ad.m4
11651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11652 // val ^ (-1 ^ (val >> shift)) ==> eonw
11653 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11654                          iRegIorL2I src1, iRegIorL2I src2,
11655                          immI src3, immI_M1 src4) %{
11656   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11657   ins_cost(1.9 * INSN_COST);
11658   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11659 
11660   ins_encode %{
11661     __ eonw(as_Register($dst$$reg),
11662               as_Register($src1$$reg),
11663               as_Register($src2$$reg),
11664               Assembler::ASR,
11665               $src3$$constant & 0x1f);
11666   %}
11667 
11668   ins_pipe(ialu_reg_reg_shift);
11669 %}
11670 
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 // val ^ (-1 ^ (val >> shift)) ==> eon
11674 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11675                          iRegL src1, iRegL src2,
11676                          immI src3, immL_M1 src4) %{
11677   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11678   ins_cost(1.9 * INSN_COST);
11679   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11680 
11681   ins_encode %{
11682     __ eon(as_Register($dst$$reg),
11683               as_Register($src1$$reg),
11684               as_Register($src2$$reg),
11685               Assembler::ASR,
11686               $src3$$constant & 0x3f);
11687   %}
11688 
11689   ins_pipe(ialu_reg_reg_shift);
11690 %}
11691 
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 // val ^ (-1 ^ (val ror shift)) ==> eonw
11695 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11696                          iRegIorL2I src1, iRegIorL2I src2,
11697                          immI src3, immI_M1 src4) %{
11698   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11699   ins_cost(1.9 * INSN_COST);
11700   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11701 
11702   ins_encode %{
11703     __ eonw(as_Register($dst$$reg),
11704               as_Register($src1$$reg),
11705               as_Register($src2$$reg),
11706               Assembler::ROR,
11707               $src3$$constant & 0x1f);
11708   %}
11709 
11710   ins_pipe(ialu_reg_reg_shift);
11711 %}
11712 
11713 // This pattern is automatically generated from aarch64_ad.m4
11714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11715 // val ^ (-1 ^ (val ror shift)) ==> eon
11716 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11717                          iRegL src1, iRegL src2,
11718                          immI src3, immL_M1 src4) %{
11719   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11720   ins_cost(1.9 * INSN_COST);
11721   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11722 
11723   ins_encode %{
11724     __ eon(as_Register($dst$$reg),
11725               as_Register($src1$$reg),
11726               as_Register($src2$$reg),
11727               Assembler::ROR,
11728               $src3$$constant & 0x3f);
11729   %}
11730 
11731   ins_pipe(ialu_reg_reg_shift);
11732 %}
11733 
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 // val ^ (-1 ^ (val << shift)) ==> eonw
11737 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11738                          iRegIorL2I src1, iRegIorL2I src2,
11739                          immI src3, immI_M1 src4) %{
11740   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11741   ins_cost(1.9 * INSN_COST);
11742   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11743 
11744   ins_encode %{
11745     __ eonw(as_Register($dst$$reg),
11746               as_Register($src1$$reg),
11747               as_Register($src2$$reg),
11748               Assembler::LSL,
11749               $src3$$constant & 0x1f);
11750   %}
11751 
11752   ins_pipe(ialu_reg_reg_shift);
11753 %}
11754 
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 // val ^ (-1 ^ (val << shift)) ==> eon
11758 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11759                          iRegL src1, iRegL src2,
11760                          immI src3, immL_M1 src4) %{
11761   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11762   ins_cost(1.9 * INSN_COST);
11763   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11764 
11765   ins_encode %{
11766     __ eon(as_Register($dst$$reg),
11767               as_Register($src1$$reg),
11768               as_Register($src2$$reg),
11769               Assembler::LSL,
11770               $src3$$constant & 0x3f);
11771   %}
11772 
11773   ins_pipe(ialu_reg_reg_shift);
11774 %}
11775 
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 // val | (-1 ^ (val >>> shift)) ==> ornw
11779 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11780                          iRegIorL2I src1, iRegIorL2I src2,
11781                          immI src3, immI_M1 src4) %{
11782   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11783   ins_cost(1.9 * INSN_COST);
11784   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11785 
11786   ins_encode %{
11787     __ ornw(as_Register($dst$$reg),
11788               as_Register($src1$$reg),
11789               as_Register($src2$$reg),
11790               Assembler::LSR,
11791               $src3$$constant & 0x1f);
11792   %}
11793 
11794   ins_pipe(ialu_reg_reg_shift);
11795 %}
11796 
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 // val | (-1 ^ (val >>> shift)) ==> orn
11800 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11801                          iRegL src1, iRegL src2,
11802                          immI src3, immL_M1 src4) %{
11803   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11804   ins_cost(1.9 * INSN_COST);
11805   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11806 
11807   ins_encode %{
11808     __ orn(as_Register($dst$$reg),
11809               as_Register($src1$$reg),
11810               as_Register($src2$$reg),
11811               Assembler::LSR,
11812               $src3$$constant & 0x3f);
11813   %}
11814 
11815   ins_pipe(ialu_reg_reg_shift);
11816 %}
11817 
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 // val | (-1 ^ (val >> shift)) ==> ornw
11821 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11822                          iRegIorL2I src1, iRegIorL2I src2,
11823                          immI src3, immI_M1 src4) %{
11824   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11825   ins_cost(1.9 * INSN_COST);
11826   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11827 
11828   ins_encode %{
11829     __ ornw(as_Register($dst$$reg),
11830               as_Register($src1$$reg),
11831               as_Register($src2$$reg),
11832               Assembler::ASR,
11833               $src3$$constant & 0x1f);
11834   %}
11835 
11836   ins_pipe(ialu_reg_reg_shift);
11837 %}
11838 
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 // val | (-1 ^ (val >> shift)) ==> orn
11842 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11843                          iRegL src1, iRegL src2,
11844                          immI src3, immL_M1 src4) %{
11845   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11846   ins_cost(1.9 * INSN_COST);
11847   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11848 
11849   ins_encode %{
11850     __ orn(as_Register($dst$$reg),
11851               as_Register($src1$$reg),
11852               as_Register($src2$$reg),
11853               Assembler::ASR,
11854               $src3$$constant & 0x3f);
11855   %}
11856 
11857   ins_pipe(ialu_reg_reg_shift);
11858 %}
11859 
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 // val | (-1 ^ (val ror shift)) ==> ornw
11863 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11864                          iRegIorL2I src1, iRegIorL2I src2,
11865                          immI src3, immI_M1 src4) %{
11866   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11867   ins_cost(1.9 * INSN_COST);
11868   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11869 
11870   ins_encode %{
11871     __ ornw(as_Register($dst$$reg),
11872               as_Register($src1$$reg),
11873               as_Register($src2$$reg),
11874               Assembler::ROR,
11875               $src3$$constant & 0x1f);
11876   %}
11877 
11878   ins_pipe(ialu_reg_reg_shift);
11879 %}
11880 
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 // val | (-1 ^ (val ror shift)) ==> orn
11884 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11885                          iRegL src1, iRegL src2,
11886                          immI src3, immL_M1 src4) %{
11887   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11888   ins_cost(1.9 * INSN_COST);
11889   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11890 
11891   ins_encode %{
11892     __ orn(as_Register($dst$$reg),
11893               as_Register($src1$$reg),
11894               as_Register($src2$$reg),
11895               Assembler::ROR,
11896               $src3$$constant & 0x3f);
11897   %}
11898 
11899   ins_pipe(ialu_reg_reg_shift);
11900 %}
11901 
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 // val | (-1 ^ (val << shift)) ==> ornw
11905 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11906                          iRegIorL2I src1, iRegIorL2I src2,
11907                          immI src3, immI_M1 src4) %{
11908   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11909   ins_cost(1.9 * INSN_COST);
11910   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11911 
11912   ins_encode %{
11913     __ ornw(as_Register($dst$$reg),
11914               as_Register($src1$$reg),
11915               as_Register($src2$$reg),
11916               Assembler::LSL,
11917               $src3$$constant & 0x1f);
11918   %}
11919 
11920   ins_pipe(ialu_reg_reg_shift);
11921 %}
11922 
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 // val | (-1 ^ (val << shift)) ==> orn
11926 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11927                          iRegL src1, iRegL src2,
11928                          immI src3, immL_M1 src4) %{
11929   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11930   ins_cost(1.9 * INSN_COST);
11931   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11932 
11933   ins_encode %{
11934     __ orn(as_Register($dst$$reg),
11935               as_Register($src1$$reg),
11936               as_Register($src2$$reg),
11937               Assembler::LSL,
11938               $src3$$constant & 0x3f);
11939   %}
11940 
11941   ins_pipe(ialu_reg_reg_shift);
11942 %}
11943 
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11947                          iRegIorL2I src1, iRegIorL2I src2,
11948                          immI src3) %{
11949   match(Set dst (AndI src1 (URShiftI src2 src3)));
11950 
11951   ins_cost(1.9 * INSN_COST);
11952   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11953 
11954   ins_encode %{
11955     __ andw(as_Register($dst$$reg),
11956               as_Register($src1$$reg),
11957               as_Register($src2$$reg),
11958               Assembler::LSR,
11959               $src3$$constant & 0x1f);
11960   %}
11961 
11962   ins_pipe(ialu_reg_reg_shift);
11963 %}
11964 
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11968                          iRegL src1, iRegL src2,
11969                          immI src3) %{
11970   match(Set dst (AndL src1 (URShiftL src2 src3)));
11971 
11972   ins_cost(1.9 * INSN_COST);
11973   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11974 
11975   ins_encode %{
11976     __ andr(as_Register($dst$$reg),
11977               as_Register($src1$$reg),
11978               as_Register($src2$$reg),
11979               Assembler::LSR,
11980               $src3$$constant & 0x3f);
11981   %}
11982 
11983   ins_pipe(ialu_reg_reg_shift);
11984 %}
11985 
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11989                          iRegIorL2I src1, iRegIorL2I src2,
11990                          immI src3) %{
11991   match(Set dst (AndI src1 (RShiftI src2 src3)));
11992 
11993   ins_cost(1.9 * INSN_COST);
11994   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11995 
11996   ins_encode %{
11997     __ andw(as_Register($dst$$reg),
11998               as_Register($src1$$reg),
11999               as_Register($src2$$reg),
12000               Assembler::ASR,
12001               $src3$$constant & 0x1f);
12002   %}
12003 
12004   ins_pipe(ialu_reg_reg_shift);
12005 %}
12006 
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12010                          iRegL src1, iRegL src2,
12011                          immI src3) %{
12012   match(Set dst (AndL src1 (RShiftL src2 src3)));
12013 
12014   ins_cost(1.9 * INSN_COST);
12015   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12016 
12017   ins_encode %{
12018     __ andr(as_Register($dst$$reg),
12019               as_Register($src1$$reg),
12020               as_Register($src2$$reg),
12021               Assembler::ASR,
12022               $src3$$constant & 0x3f);
12023   %}
12024 
12025   ins_pipe(ialu_reg_reg_shift);
12026 %}
12027 
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12031                          iRegIorL2I src1, iRegIorL2I src2,
12032                          immI src3) %{
12033   match(Set dst (AndI src1 (LShiftI src2 src3)));
12034 
12035   ins_cost(1.9 * INSN_COST);
12036   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12037 
12038   ins_encode %{
12039     __ andw(as_Register($dst$$reg),
12040               as_Register($src1$$reg),
12041               as_Register($src2$$reg),
12042               Assembler::LSL,
12043               $src3$$constant & 0x1f);
12044   %}
12045 
12046   ins_pipe(ialu_reg_reg_shift);
12047 %}
12048 
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12052                          iRegL src1, iRegL src2,
12053                          immI src3) %{
12054   match(Set dst (AndL src1 (LShiftL src2 src3)));
12055 
12056   ins_cost(1.9 * INSN_COST);
12057   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12058 
12059   ins_encode %{
12060     __ andr(as_Register($dst$$reg),
12061               as_Register($src1$$reg),
12062               as_Register($src2$$reg),
12063               Assembler::LSL,
12064               $src3$$constant & 0x3f);
12065   %}
12066 
12067   ins_pipe(ialu_reg_reg_shift);
12068 %}
12069 
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12073                          iRegIorL2I src1, iRegIorL2I src2,
12074                          immI src3) %{
12075   match(Set dst (AndI src1 (RotateRight src2 src3)));
12076 
12077   ins_cost(1.9 * INSN_COST);
12078   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12079 
12080   ins_encode %{
12081     __ andw(as_Register($dst$$reg),
12082               as_Register($src1$$reg),
12083               as_Register($src2$$reg),
12084               Assembler::ROR,
12085               $src3$$constant & 0x1f);
12086   %}
12087 
12088   ins_pipe(ialu_reg_reg_shift);
12089 %}
12090 
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12094                          iRegL src1, iRegL src2,
12095                          immI src3) %{
12096   match(Set dst (AndL src1 (RotateRight src2 src3)));
12097 
12098   ins_cost(1.9 * INSN_COST);
12099   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12100 
12101   ins_encode %{
12102     __ andr(as_Register($dst$$reg),
12103               as_Register($src1$$reg),
12104               as_Register($src2$$reg),
12105               Assembler::ROR,
12106               $src3$$constant & 0x3f);
12107   %}
12108 
12109   ins_pipe(ialu_reg_reg_shift);
12110 %}
12111 
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12115                          iRegIorL2I src1, iRegIorL2I src2,
12116                          immI src3) %{
12117   match(Set dst (XorI src1 (URShiftI src2 src3)));
12118 
12119   ins_cost(1.9 * INSN_COST);
12120   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12121 
12122   ins_encode %{
12123     __ eorw(as_Register($dst$$reg),
12124               as_Register($src1$$reg),
12125               as_Register($src2$$reg),
12126               Assembler::LSR,
12127               $src3$$constant & 0x1f);
12128   %}
12129 
12130   ins_pipe(ialu_reg_reg_shift);
12131 %}
12132 
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12136                          iRegL src1, iRegL src2,
12137                          immI src3) %{
12138   match(Set dst (XorL src1 (URShiftL src2 src3)));
12139 
12140   ins_cost(1.9 * INSN_COST);
12141   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12142 
12143   ins_encode %{
12144     __ eor(as_Register($dst$$reg),
12145               as_Register($src1$$reg),
12146               as_Register($src2$$reg),
12147               Assembler::LSR,
12148               $src3$$constant & 0x3f);
12149   %}
12150 
12151   ins_pipe(ialu_reg_reg_shift);
12152 %}
12153 
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12157                          iRegIorL2I src1, iRegIorL2I src2,
12158                          immI src3) %{
12159   match(Set dst (XorI src1 (RShiftI src2 src3)));
12160 
12161   ins_cost(1.9 * INSN_COST);
12162   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12163 
12164   ins_encode %{
12165     __ eorw(as_Register($dst$$reg),
12166               as_Register($src1$$reg),
12167               as_Register($src2$$reg),
12168               Assembler::ASR,
12169               $src3$$constant & 0x1f);
12170   %}
12171 
12172   ins_pipe(ialu_reg_reg_shift);
12173 %}
12174 
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12178                          iRegL src1, iRegL src2,
12179                          immI src3) %{
12180   match(Set dst (XorL src1 (RShiftL src2 src3)));
12181 
12182   ins_cost(1.9 * INSN_COST);
12183   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12184 
12185   ins_encode %{
12186     __ eor(as_Register($dst$$reg),
12187               as_Register($src1$$reg),
12188               as_Register($src2$$reg),
12189               Assembler::ASR,
12190               $src3$$constant & 0x3f);
12191   %}
12192 
12193   ins_pipe(ialu_reg_reg_shift);
12194 %}
12195 
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12199                          iRegIorL2I src1, iRegIorL2I src2,
12200                          immI src3) %{
12201   match(Set dst (XorI src1 (LShiftI src2 src3)));
12202 
12203   ins_cost(1.9 * INSN_COST);
12204   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12205 
12206   ins_encode %{
12207     __ eorw(as_Register($dst$$reg),
12208               as_Register($src1$$reg),
12209               as_Register($src2$$reg),
12210               Assembler::LSL,
12211               $src3$$constant & 0x1f);
12212   %}
12213 
12214   ins_pipe(ialu_reg_reg_shift);
12215 %}
12216 
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12220                          iRegL src1, iRegL src2,
12221                          immI src3) %{
12222   match(Set dst (XorL src1 (LShiftL src2 src3)));
12223 
12224   ins_cost(1.9 * INSN_COST);
12225   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12226 
12227   ins_encode %{
12228     __ eor(as_Register($dst$$reg),
12229               as_Register($src1$$reg),
12230               as_Register($src2$$reg),
12231               Assembler::LSL,
12232               $src3$$constant & 0x3f);
12233   %}
12234 
12235   ins_pipe(ialu_reg_reg_shift);
12236 %}
12237 
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12241                          iRegIorL2I src1, iRegIorL2I src2,
12242                          immI src3) %{
12243   match(Set dst (XorI src1 (RotateRight src2 src3)));
12244 
12245   ins_cost(1.9 * INSN_COST);
12246   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12247 
12248   ins_encode %{
12249     __ eorw(as_Register($dst$$reg),
12250               as_Register($src1$$reg),
12251               as_Register($src2$$reg),
12252               Assembler::ROR,
12253               $src3$$constant & 0x1f);
12254   %}
12255 
12256   ins_pipe(ialu_reg_reg_shift);
12257 %}
12258 
12259 // This pattern is automatically generated from aarch64_ad.m4
12260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12261 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12262                          iRegL src1, iRegL src2,
12263                          immI src3) %{
12264   match(Set dst (XorL src1 (RotateRight src2 src3)));
12265 
12266   ins_cost(1.9 * INSN_COST);
12267   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12268 
12269   ins_encode %{
12270     __ eor(as_Register($dst$$reg),
12271               as_Register($src1$$reg),
12272               as_Register($src2$$reg),
12273               Assembler::ROR,
12274               $src3$$constant & 0x3f);
12275   %}
12276 
12277   ins_pipe(ialu_reg_reg_shift);
12278 %}
12279 
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12283                          iRegIorL2I src1, iRegIorL2I src2,
12284                          immI src3) %{
12285   match(Set dst (OrI src1 (URShiftI src2 src3)));
12286 
12287   ins_cost(1.9 * INSN_COST);
12288   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12289 
12290   ins_encode %{
12291     __ orrw(as_Register($dst$$reg),
12292               as_Register($src1$$reg),
12293               as_Register($src2$$reg),
12294               Assembler::LSR,
12295               $src3$$constant & 0x1f);
12296   %}
12297 
12298   ins_pipe(ialu_reg_reg_shift);
12299 %}
12300 
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12304                          iRegL src1, iRegL src2,
12305                          immI src3) %{
12306   match(Set dst (OrL src1 (URShiftL src2 src3)));
12307 
12308   ins_cost(1.9 * INSN_COST);
12309   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12310 
12311   ins_encode %{
12312     __ orr(as_Register($dst$$reg),
12313               as_Register($src1$$reg),
12314               as_Register($src2$$reg),
12315               Assembler::LSR,
12316               $src3$$constant & 0x3f);
12317   %}
12318 
12319   ins_pipe(ialu_reg_reg_shift);
12320 %}
12321 
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12325                          iRegIorL2I src1, iRegIorL2I src2,
12326                          immI src3) %{
12327   match(Set dst (OrI src1 (RShiftI src2 src3)));
12328 
12329   ins_cost(1.9 * INSN_COST);
12330   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12331 
12332   ins_encode %{
12333     __ orrw(as_Register($dst$$reg),
12334               as_Register($src1$$reg),
12335               as_Register($src2$$reg),
12336               Assembler::ASR,
12337               $src3$$constant & 0x1f);
12338   %}
12339 
12340   ins_pipe(ialu_reg_reg_shift);
12341 %}
12342 
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12346                          iRegL src1, iRegL src2,
12347                          immI src3) %{
12348   match(Set dst (OrL src1 (RShiftL src2 src3)));
12349 
12350   ins_cost(1.9 * INSN_COST);
12351   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12352 
12353   ins_encode %{
12354     __ orr(as_Register($dst$$reg),
12355               as_Register($src1$$reg),
12356               as_Register($src2$$reg),
12357               Assembler::ASR,
12358               $src3$$constant & 0x3f);
12359   %}
12360 
12361   ins_pipe(ialu_reg_reg_shift);
12362 %}
12363 
12364 // This pattern is automatically generated from aarch64_ad.m4
12365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12366 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12367                          iRegIorL2I src1, iRegIorL2I src2,
12368                          immI src3) %{
12369   match(Set dst (OrI src1 (LShiftI src2 src3)));
12370 
12371   ins_cost(1.9 * INSN_COST);
12372   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12373 
12374   ins_encode %{
12375     __ orrw(as_Register($dst$$reg),
12376               as_Register($src1$$reg),
12377               as_Register($src2$$reg),
12378               Assembler::LSL,
12379               $src3$$constant & 0x1f);
12380   %}
12381 
12382   ins_pipe(ialu_reg_reg_shift);
12383 %}
12384 
12385 // This pattern is automatically generated from aarch64_ad.m4
12386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12387 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12388                          iRegL src1, iRegL src2,
12389                          immI src3) %{
12390   match(Set dst (OrL src1 (LShiftL src2 src3)));
12391 
12392   ins_cost(1.9 * INSN_COST);
12393   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12394 
12395   ins_encode %{
12396     __ orr(as_Register($dst$$reg),
12397               as_Register($src1$$reg),
12398               as_Register($src2$$reg),
12399               Assembler::LSL,
12400               $src3$$constant & 0x3f);
12401   %}
12402 
12403   ins_pipe(ialu_reg_reg_shift);
12404 %}
12405 
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12409                          iRegIorL2I src1, iRegIorL2I src2,
12410                          immI src3) %{
12411   match(Set dst (OrI src1 (RotateRight src2 src3)));
12412 
12413   ins_cost(1.9 * INSN_COST);
12414   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12415 
12416   ins_encode %{
12417     __ orrw(as_Register($dst$$reg),
12418               as_Register($src1$$reg),
12419               as_Register($src2$$reg),
12420               Assembler::ROR,
12421               $src3$$constant & 0x1f);
12422   %}
12423 
12424   ins_pipe(ialu_reg_reg_shift);
12425 %}
12426 
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12430                          iRegL src1, iRegL src2,
12431                          immI src3) %{
12432   match(Set dst (OrL src1 (RotateRight src2 src3)));
12433 
12434   ins_cost(1.9 * INSN_COST);
12435   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12436 
12437   ins_encode %{
12438     __ orr(as_Register($dst$$reg),
12439               as_Register($src1$$reg),
12440               as_Register($src2$$reg),
12441               Assembler::ROR,
12442               $src3$$constant & 0x3f);
12443   %}
12444 
12445   ins_pipe(ialu_reg_reg_shift);
12446 %}
12447 
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12451                          iRegIorL2I src1, iRegIorL2I src2,
12452                          immI src3) %{
12453   match(Set dst (AddI src1 (URShiftI src2 src3)));
12454 
12455   ins_cost(1.9 * INSN_COST);
12456   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12457 
12458   ins_encode %{
12459     __ addw(as_Register($dst$$reg),
12460               as_Register($src1$$reg),
12461               as_Register($src2$$reg),
12462               Assembler::LSR,
12463               $src3$$constant & 0x1f);
12464   %}
12465 
12466   ins_pipe(ialu_reg_reg_shift);
12467 %}
12468 
12469 // This pattern is automatically generated from aarch64_ad.m4
12470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12471 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12472                          iRegL src1, iRegL src2,
12473                          immI src3) %{
12474   match(Set dst (AddL src1 (URShiftL src2 src3)));
12475 
12476   ins_cost(1.9 * INSN_COST);
12477   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12478 
12479   ins_encode %{
12480     __ add(as_Register($dst$$reg),
12481               as_Register($src1$$reg),
12482               as_Register($src2$$reg),
12483               Assembler::LSR,
12484               $src3$$constant & 0x3f);
12485   %}
12486 
12487   ins_pipe(ialu_reg_reg_shift);
12488 %}
12489 
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12493                          iRegIorL2I src1, iRegIorL2I src2,
12494                          immI src3) %{
12495   match(Set dst (AddI src1 (RShiftI src2 src3)));
12496 
12497   ins_cost(1.9 * INSN_COST);
12498   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12499 
12500   ins_encode %{
12501     __ addw(as_Register($dst$$reg),
12502               as_Register($src1$$reg),
12503               as_Register($src2$$reg),
12504               Assembler::ASR,
12505               $src3$$constant & 0x1f);
12506   %}
12507 
12508   ins_pipe(ialu_reg_reg_shift);
12509 %}
12510 
12511 // This pattern is automatically generated from aarch64_ad.m4
12512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12513 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12514                          iRegL src1, iRegL src2,
12515                          immI src3) %{
12516   match(Set dst (AddL src1 (RShiftL src2 src3)));
12517 
12518   ins_cost(1.9 * INSN_COST);
12519   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12520 
12521   ins_encode %{
12522     __ add(as_Register($dst$$reg),
12523               as_Register($src1$$reg),
12524               as_Register($src2$$reg),
12525               Assembler::ASR,
12526               $src3$$constant & 0x3f);
12527   %}
12528 
12529   ins_pipe(ialu_reg_reg_shift);
12530 %}
12531 
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12535                          iRegIorL2I src1, iRegIorL2I src2,
12536                          immI src3) %{
12537   match(Set dst (AddI src1 (LShiftI src2 src3)));
12538 
12539   ins_cost(1.9 * INSN_COST);
12540   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12541 
12542   ins_encode %{
12543     __ addw(as_Register($dst$$reg),
12544               as_Register($src1$$reg),
12545               as_Register($src2$$reg),
12546               Assembler::LSL,
12547               $src3$$constant & 0x1f);
12548   %}
12549 
12550   ins_pipe(ialu_reg_reg_shift);
12551 %}
12552 
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12556                          iRegL src1, iRegL src2,
12557                          immI src3) %{
12558   match(Set dst (AddL src1 (LShiftL src2 src3)));
12559 
12560   ins_cost(1.9 * INSN_COST);
12561   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12562 
12563   ins_encode %{
12564     __ add(as_Register($dst$$reg),
12565               as_Register($src1$$reg),
12566               as_Register($src2$$reg),
12567               Assembler::LSL,
12568               $src3$$constant & 0x3f);
12569   %}
12570 
12571   ins_pipe(ialu_reg_reg_shift);
12572 %}
12573 
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12577                          iRegIorL2I src1, iRegIorL2I src2,
12578                          immI src3) %{
12579   match(Set dst (SubI src1 (URShiftI src2 src3)));
12580 
12581   ins_cost(1.9 * INSN_COST);
12582   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12583 
12584   ins_encode %{
12585     __ subw(as_Register($dst$$reg),
12586               as_Register($src1$$reg),
12587               as_Register($src2$$reg),
12588               Assembler::LSR,
12589               $src3$$constant & 0x1f);
12590   %}
12591 
12592   ins_pipe(ialu_reg_reg_shift);
12593 %}
12594 
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12598                          iRegL src1, iRegL src2,
12599                          immI src3) %{
12600   match(Set dst (SubL src1 (URShiftL src2 src3)));
12601 
12602   ins_cost(1.9 * INSN_COST);
12603   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12604 
12605   ins_encode %{
12606     __ sub(as_Register($dst$$reg),
12607               as_Register($src1$$reg),
12608               as_Register($src2$$reg),
12609               Assembler::LSR,
12610               $src3$$constant & 0x3f);
12611   %}
12612 
12613   ins_pipe(ialu_reg_reg_shift);
12614 %}
12615 
12616 // This pattern is automatically generated from aarch64_ad.m4
12617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12618 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12619                          iRegIorL2I src1, iRegIorL2I src2,
12620                          immI src3) %{
12621   match(Set dst (SubI src1 (RShiftI src2 src3)));
12622 
12623   ins_cost(1.9 * INSN_COST);
12624   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12625 
12626   ins_encode %{
12627     __ subw(as_Register($dst$$reg),
12628               as_Register($src1$$reg),
12629               as_Register($src2$$reg),
12630               Assembler::ASR,
12631               $src3$$constant & 0x1f);
12632   %}
12633 
12634   ins_pipe(ialu_reg_reg_shift);
12635 %}
12636 
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12640                          iRegL src1, iRegL src2,
12641                          immI src3) %{
12642   match(Set dst (SubL src1 (RShiftL src2 src3)));
12643 
12644   ins_cost(1.9 * INSN_COST);
12645   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12646 
12647   ins_encode %{
12648     __ sub(as_Register($dst$$reg),
12649               as_Register($src1$$reg),
12650               as_Register($src2$$reg),
12651               Assembler::ASR,
12652               $src3$$constant & 0x3f);
12653   %}
12654 
12655   ins_pipe(ialu_reg_reg_shift);
12656 %}
12657 
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12661                          iRegIorL2I src1, iRegIorL2I src2,
12662                          immI src3) %{
12663   match(Set dst (SubI src1 (LShiftI src2 src3)));
12664 
12665   ins_cost(1.9 * INSN_COST);
12666   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12667 
12668   ins_encode %{
12669     __ subw(as_Register($dst$$reg),
12670               as_Register($src1$$reg),
12671               as_Register($src2$$reg),
12672               Assembler::LSL,
12673               $src3$$constant & 0x1f);
12674   %}
12675 
12676   ins_pipe(ialu_reg_reg_shift);
12677 %}
12678 
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12682                          iRegL src1, iRegL src2,
12683                          immI src3) %{
12684   match(Set dst (SubL src1 (LShiftL src2 src3)));
12685 
12686   ins_cost(1.9 * INSN_COST);
12687   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12688 
12689   ins_encode %{
12690     __ sub(as_Register($dst$$reg),
12691               as_Register($src1$$reg),
12692               as_Register($src2$$reg),
12693               Assembler::LSL,
12694               $src3$$constant & 0x3f);
12695   %}
12696 
12697   ins_pipe(ialu_reg_reg_shift);
12698 %}
12699 
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 
12703 // Shift Left followed by Shift Right.
12704 // This idiom is used by the compiler for the i2b bytecode etc.
12705 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12706 %{
12707   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12708   ins_cost(INSN_COST * 2);
12709   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12710   ins_encode %{
12711     int lshift = $lshift_count$$constant & 63;
12712     int rshift = $rshift_count$$constant & 63;
12713     int s = 63 - lshift;
12714     int r = (rshift - lshift) & 63;
12715     __ sbfm(as_Register($dst$$reg),
12716             as_Register($src$$reg),
12717             r, s);
12718   %}
12719 
12720   ins_pipe(ialu_reg_shift);
12721 %}
12722 
12723 // This pattern is automatically generated from aarch64_ad.m4
12724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12725 
12726 // Shift Left followed by Shift Right.
12727 // This idiom is used by the compiler for the i2b bytecode etc.
12728 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12729 %{
12730   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12731   ins_cost(INSN_COST * 2);
12732   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12733   ins_encode %{
12734     int lshift = $lshift_count$$constant & 31;
12735     int rshift = $rshift_count$$constant & 31;
12736     int s = 31 - lshift;
12737     int r = (rshift - lshift) & 31;
12738     __ sbfmw(as_Register($dst$$reg),
12739             as_Register($src$$reg),
12740             r, s);
12741   %}
12742 
12743   ins_pipe(ialu_reg_shift);
12744 %}
12745 
12746 // This pattern is automatically generated from aarch64_ad.m4
12747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12748 
12749 // Shift Left followed by Shift Right.
12750 // This idiom is used by the compiler for the i2b bytecode etc.
12751 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12752 %{
12753   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12754   ins_cost(INSN_COST * 2);
12755   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12756   ins_encode %{
12757     int lshift = $lshift_count$$constant & 63;
12758     int rshift = $rshift_count$$constant & 63;
12759     int s = 63 - lshift;
12760     int r = (rshift - lshift) & 63;
12761     __ ubfm(as_Register($dst$$reg),
12762             as_Register($src$$reg),
12763             r, s);
12764   %}
12765 
12766   ins_pipe(ialu_reg_shift);
12767 %}
12768 
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 
12772 // Shift Left followed by Shift Right.
12773 // This idiom is used by the compiler for the i2b bytecode etc.
12774 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12775 %{
12776   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12777   ins_cost(INSN_COST * 2);
12778   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12779   ins_encode %{
12780     int lshift = $lshift_count$$constant & 31;
12781     int rshift = $rshift_count$$constant & 31;
12782     int s = 31 - lshift;
12783     int r = (rshift - lshift) & 31;
12784     __ ubfmw(as_Register($dst$$reg),
12785             as_Register($src$$reg),
12786             r, s);
12787   %}
12788 
12789   ins_pipe(ialu_reg_shift);
12790 %}
12791 
12792 // Bitfield extract with shift & mask
12793 
12794 // This pattern is automatically generated from aarch64_ad.m4
12795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12796 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12797 %{
12798   match(Set dst (AndI (URShiftI src rshift) mask));
12799   // Make sure we are not going to exceed what ubfxw can do.
12800   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12801 
12802   ins_cost(INSN_COST);
12803   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12804   ins_encode %{
12805     int rshift = $rshift$$constant & 31;
12806     intptr_t mask = $mask$$constant;
12807     int width = exact_log2(mask+1);
12808     __ ubfxw(as_Register($dst$$reg),
12809             as_Register($src$$reg), rshift, width);
12810   %}
12811   ins_pipe(ialu_reg_shift);
12812 %}
12813 
12814 // This pattern is automatically generated from aarch64_ad.m4
12815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12816 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12817 %{
12818   match(Set dst (AndL (URShiftL src rshift) mask));
12819   // Make sure we are not going to exceed what ubfx can do.
12820   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12821 
12822   ins_cost(INSN_COST);
12823   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12824   ins_encode %{
12825     int rshift = $rshift$$constant & 63;
12826     intptr_t mask = $mask$$constant;
12827     int width = exact_log2_long(mask+1);
12828     __ ubfx(as_Register($dst$$reg),
12829             as_Register($src$$reg), rshift, width);
12830   %}
12831   ins_pipe(ialu_reg_shift);
12832 %}
12833 
12834 
12835 // This pattern is automatically generated from aarch64_ad.m4
12836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12837 
12838 // We can use ubfx when extending an And with a mask when we know mask
12839 // is positive.  We know that because immI_bitmask guarantees it.
12840 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12841 %{
12842   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12843   // Make sure we are not going to exceed what ubfxw can do.
12844   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12845 
12846   ins_cost(INSN_COST * 2);
12847   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12848   ins_encode %{
12849     int rshift = $rshift$$constant & 31;
12850     intptr_t mask = $mask$$constant;
12851     int width = exact_log2(mask+1);
12852     __ ubfx(as_Register($dst$$reg),
12853             as_Register($src$$reg), rshift, width);
12854   %}
12855   ins_pipe(ialu_reg_shift);
12856 %}
12857 
12858 
12859 // This pattern is automatically generated from aarch64_ad.m4
12860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12861 
12862 // We can use ubfiz when masking by a positive number and then left shifting the result.
12863 // We know that the mask is positive because immI_bitmask guarantees it.
12864 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12865 %{
12866   match(Set dst (LShiftI (AndI src mask) lshift));
12867   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12868 
12869   ins_cost(INSN_COST);
12870   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12871   ins_encode %{
12872     int lshift = $lshift$$constant & 31;
12873     intptr_t mask = $mask$$constant;
12874     int width = exact_log2(mask+1);
12875     __ ubfizw(as_Register($dst$$reg),
12876           as_Register($src$$reg), lshift, width);
12877   %}
12878   ins_pipe(ialu_reg_shift);
12879 %}
12880 
12881 // This pattern is automatically generated from aarch64_ad.m4
12882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12883 
12884 // We can use ubfiz when masking by a positive number and then left shifting the result.
12885 // We know that the mask is positive because immL_bitmask guarantees it.
12886 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12887 %{
12888   match(Set dst (LShiftL (AndL src mask) lshift));
12889   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12890 
12891   ins_cost(INSN_COST);
12892   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12893   ins_encode %{
12894     int lshift = $lshift$$constant & 63;
12895     intptr_t mask = $mask$$constant;
12896     int width = exact_log2_long(mask+1);
12897     __ ubfiz(as_Register($dst$$reg),
12898           as_Register($src$$reg), lshift, width);
12899   %}
12900   ins_pipe(ialu_reg_shift);
12901 %}
12902 
12903 // This pattern is automatically generated from aarch64_ad.m4
12904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12905 
12906 // We can use ubfiz when masking by a positive number and then left shifting the result.
12907 // We know that the mask is positive because immI_bitmask guarantees it.
12908 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12909 %{
12910   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12911   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12912 
12913   ins_cost(INSN_COST);
12914   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12915   ins_encode %{
12916     int lshift = $lshift$$constant & 31;
12917     intptr_t mask = $mask$$constant;
12918     int width = exact_log2(mask+1);
12919     __ ubfizw(as_Register($dst$$reg),
12920           as_Register($src$$reg), lshift, width);
12921   %}
12922   ins_pipe(ialu_reg_shift);
12923 %}
12924 
12925 // This pattern is automatically generated from aarch64_ad.m4
12926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12927 
12928 // We can use ubfiz when masking by a positive number and then left shifting the result.
12929 // We know that the mask is positive because immL_bitmask guarantees it.
12930 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12931 %{
12932   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12933   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12934 
12935   ins_cost(INSN_COST);
12936   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12937   ins_encode %{
12938     int lshift = $lshift$$constant & 63;
12939     intptr_t mask = $mask$$constant;
12940     int width = exact_log2_long(mask+1);
12941     __ ubfiz(as_Register($dst$$reg),
12942           as_Register($src$$reg), lshift, width);
12943   %}
12944   ins_pipe(ialu_reg_shift);
12945 %}
12946 
12947 
12948 // This pattern is automatically generated from aarch64_ad.m4
12949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12950 
12951 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12952 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12953 %{
12954   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12955   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12956 
12957   ins_cost(INSN_COST);
12958   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12959   ins_encode %{
12960     int lshift = $lshift$$constant & 63;
12961     intptr_t mask = $mask$$constant;
12962     int width = exact_log2(mask+1);
12963     __ ubfiz(as_Register($dst$$reg),
12964              as_Register($src$$reg), lshift, width);
12965   %}
12966   ins_pipe(ialu_reg_shift);
12967 %}
12968 
12969 // This pattern is automatically generated from aarch64_ad.m4
12970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12971 
12972 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12973 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12974 %{
12975   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12976   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12977 
12978   ins_cost(INSN_COST);
12979   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12980   ins_encode %{
12981     int lshift = $lshift$$constant & 31;
12982     intptr_t mask = $mask$$constant;
12983     int width = exact_log2(mask+1);
12984     __ ubfiz(as_Register($dst$$reg),
12985              as_Register($src$$reg), lshift, width);
12986   %}
12987   ins_pipe(ialu_reg_shift);
12988 %}
12989 
12990 // This pattern is automatically generated from aarch64_ad.m4
12991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12992 
12993 // Can skip int2long conversions after AND with small bitmask
12994 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12995 %{
12996   match(Set dst (ConvI2L (AndI src msk)));
12997   ins_cost(INSN_COST);
12998   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12999   ins_encode %{
13000     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13001   %}
13002   ins_pipe(ialu_reg_shift);
13003 %}
13004 
13005 
13006 // Rotations
13007 // This pattern is automatically generated from aarch64_ad.m4
13008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13009 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13010 %{
13011   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13012   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13013 
13014   ins_cost(INSN_COST);
13015   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13016 
13017   ins_encode %{
13018     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13019             $rshift$$constant & 63);
13020   %}
13021   ins_pipe(ialu_reg_reg_extr);
13022 %}
13023 
13024 
13025 // This pattern is automatically generated from aarch64_ad.m4
13026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13027 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13028 %{
13029   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13030   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13031 
13032   ins_cost(INSN_COST);
13033   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13034 
13035   ins_encode %{
13036     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13037             $rshift$$constant & 31);
13038   %}
13039   ins_pipe(ialu_reg_reg_extr);
13040 %}
13041 
13042 
13043 // This pattern is automatically generated from aarch64_ad.m4
13044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13045 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13046 %{
13047   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13048   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13049 
13050   ins_cost(INSN_COST);
13051   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13052 
13053   ins_encode %{
13054     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13055             $rshift$$constant & 63);
13056   %}
13057   ins_pipe(ialu_reg_reg_extr);
13058 %}
13059 
13060 
13061 // This pattern is automatically generated from aarch64_ad.m4
13062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13063 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13064 %{
13065   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13066   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13067 
13068   ins_cost(INSN_COST);
13069   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13070 
13071   ins_encode %{
13072     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13073             $rshift$$constant & 31);
13074   %}
13075   ins_pipe(ialu_reg_reg_extr);
13076 %}
13077 
13078 
13079 // This pattern is automatically generated from aarch64_ad.m4
13080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13081 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13082 %{
13083   match(Set dst (RotateRight src shift));
13084 
13085   ins_cost(INSN_COST);
13086   format %{ "ror    $dst, $src, $shift" %}
13087 
13088   ins_encode %{
13089      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13090                $shift$$constant & 0x1f);
13091   %}
13092   ins_pipe(ialu_reg_reg_vshift);
13093 %}
13094 
13095 // This pattern is automatically generated from aarch64_ad.m4
13096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13097 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13098 %{
13099   match(Set dst (RotateRight src shift));
13100 
13101   ins_cost(INSN_COST);
13102   format %{ "ror    $dst, $src, $shift" %}
13103 
13104   ins_encode %{
13105      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13106                $shift$$constant & 0x3f);
13107   %}
13108   ins_pipe(ialu_reg_reg_vshift);
13109 %}
13110 
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13114 %{
13115   match(Set dst (RotateRight src shift));
13116 
13117   ins_cost(INSN_COST);
13118   format %{ "ror    $dst, $src, $shift" %}
13119 
13120   ins_encode %{
13121      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13122   %}
13123   ins_pipe(ialu_reg_reg_vshift);
13124 %}
13125 
13126 // This pattern is automatically generated from aarch64_ad.m4
13127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13128 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13129 %{
13130   match(Set dst (RotateRight src shift));
13131 
13132   ins_cost(INSN_COST);
13133   format %{ "ror    $dst, $src, $shift" %}
13134 
13135   ins_encode %{
13136      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13137   %}
13138   ins_pipe(ialu_reg_reg_vshift);
13139 %}
13140 
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13144 %{
13145   match(Set dst (RotateLeft src shift));
13146 
13147   ins_cost(INSN_COST);
13148   format %{ "rol    $dst, $src, $shift" %}
13149 
13150   ins_encode %{
13151      __ subw(rscratch1, zr, as_Register($shift$$reg));
13152      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13153   %}
13154   ins_pipe(ialu_reg_reg_vshift);
13155 %}
13156 
13157 // This pattern is automatically generated from aarch64_ad.m4
13158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13159 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13160 %{
13161   match(Set dst (RotateLeft src shift));
13162 
13163   ins_cost(INSN_COST);
13164   format %{ "rol    $dst, $src, $shift" %}
13165 
13166   ins_encode %{
13167      __ subw(rscratch1, zr, as_Register($shift$$reg));
13168      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13169   %}
13170   ins_pipe(ialu_reg_reg_vshift);
13171 %}
13172 
13173 
13174 // Add/subtract (extended)
13175 
13176 // This pattern is automatically generated from aarch64_ad.m4
13177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13178 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13179 %{
13180   match(Set dst (AddL src1 (ConvI2L src2)));
13181   ins_cost(INSN_COST);
13182   format %{ "add  $dst, $src1, $src2, sxtw" %}
13183 
13184    ins_encode %{
13185      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13186             as_Register($src2$$reg), ext::sxtw);
13187    %}
13188   ins_pipe(ialu_reg_reg);
13189 %}
13190 
13191 // This pattern is automatically generated from aarch64_ad.m4
13192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13193 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13194 %{
13195   match(Set dst (SubL src1 (ConvI2L src2)));
13196   ins_cost(INSN_COST);
13197   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13198 
13199    ins_encode %{
13200      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13201             as_Register($src2$$reg), ext::sxtw);
13202    %}
13203   ins_pipe(ialu_reg_reg);
13204 %}
13205 
13206 // This pattern is automatically generated from aarch64_ad.m4
13207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13208 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13209 %{
13210   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13211   ins_cost(INSN_COST);
13212   format %{ "add  $dst, $src1, $src2, sxth" %}
13213 
13214    ins_encode %{
13215      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13216             as_Register($src2$$reg), ext::sxth);
13217    %}
13218   ins_pipe(ialu_reg_reg);
13219 %}
13220 
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13224 %{
13225   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13226   ins_cost(INSN_COST);
13227   format %{ "add  $dst, $src1, $src2, sxtb" %}
13228 
13229    ins_encode %{
13230      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13231             as_Register($src2$$reg), ext::sxtb);
13232    %}
13233   ins_pipe(ialu_reg_reg);
13234 %}
13235 
13236 // This pattern is automatically generated from aarch64_ad.m4
13237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13238 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13239 %{
13240   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13241   ins_cost(INSN_COST);
13242   format %{ "add  $dst, $src1, $src2, uxtb" %}
13243 
13244    ins_encode %{
13245      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13246             as_Register($src2$$reg), ext::uxtb);
13247    %}
13248   ins_pipe(ialu_reg_reg);
13249 %}
13250 
13251 // This pattern is automatically generated from aarch64_ad.m4
13252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13253 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13254 %{
13255   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13256   ins_cost(INSN_COST);
13257   format %{ "add  $dst, $src1, $src2, sxth" %}
13258 
13259    ins_encode %{
13260      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13261             as_Register($src2$$reg), ext::sxth);
13262    %}
13263   ins_pipe(ialu_reg_reg);
13264 %}
13265 
13266 // This pattern is automatically generated from aarch64_ad.m4
13267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13268 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13269 %{
13270   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13271   ins_cost(INSN_COST);
13272   format %{ "add  $dst, $src1, $src2, sxtw" %}
13273 
13274    ins_encode %{
13275      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13276             as_Register($src2$$reg), ext::sxtw);
13277    %}
13278   ins_pipe(ialu_reg_reg);
13279 %}
13280 
13281 // This pattern is automatically generated from aarch64_ad.m4
13282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13283 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13284 %{
13285   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13286   ins_cost(INSN_COST);
13287   format %{ "add  $dst, $src1, $src2, sxtb" %}
13288 
13289    ins_encode %{
13290      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13291             as_Register($src2$$reg), ext::sxtb);
13292    %}
13293   ins_pipe(ialu_reg_reg);
13294 %}
13295 
13296 // This pattern is automatically generated from aarch64_ad.m4
13297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13298 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13299 %{
13300   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13301   ins_cost(INSN_COST);
13302   format %{ "add  $dst, $src1, $src2, uxtb" %}
13303 
13304    ins_encode %{
13305      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13306             as_Register($src2$$reg), ext::uxtb);
13307    %}
13308   ins_pipe(ialu_reg_reg);
13309 %}
13310 
13311 // This pattern is automatically generated from aarch64_ad.m4
13312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13313 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13314 %{
13315   match(Set dst (AddI src1 (AndI src2 mask)));
13316   ins_cost(INSN_COST);
13317   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13318 
13319    ins_encode %{
13320      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13321             as_Register($src2$$reg), ext::uxtb);
13322    %}
13323   ins_pipe(ialu_reg_reg);
13324 %}
13325 
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13329 %{
13330   match(Set dst (AddI src1 (AndI src2 mask)));
13331   ins_cost(INSN_COST);
13332   format %{ "addw  $dst, $src1, $src2, uxth" %}
13333 
13334    ins_encode %{
13335      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13336             as_Register($src2$$reg), ext::uxth);
13337    %}
13338   ins_pipe(ialu_reg_reg);
13339 %}
13340 
13341 // This pattern is automatically generated from aarch64_ad.m4
13342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13343 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13344 %{
13345   match(Set dst (AddL src1 (AndL src2 mask)));
13346   ins_cost(INSN_COST);
13347   format %{ "add  $dst, $src1, $src2, uxtb" %}
13348 
13349    ins_encode %{
13350      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13351             as_Register($src2$$reg), ext::uxtb);
13352    %}
13353   ins_pipe(ialu_reg_reg);
13354 %}
13355 
13356 // This pattern is automatically generated from aarch64_ad.m4
13357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13358 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13359 %{
13360   match(Set dst (AddL src1 (AndL src2 mask)));
13361   ins_cost(INSN_COST);
13362   format %{ "add  $dst, $src1, $src2, uxth" %}
13363 
13364    ins_encode %{
13365      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13366             as_Register($src2$$reg), ext::uxth);
13367    %}
13368   ins_pipe(ialu_reg_reg);
13369 %}
13370 
13371 // This pattern is automatically generated from aarch64_ad.m4
13372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13373 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13374 %{
13375   match(Set dst (AddL src1 (AndL src2 mask)));
13376   ins_cost(INSN_COST);
13377   format %{ "add  $dst, $src1, $src2, uxtw" %}
13378 
13379    ins_encode %{
13380      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13381             as_Register($src2$$reg), ext::uxtw);
13382    %}
13383   ins_pipe(ialu_reg_reg);
13384 %}
13385 
13386 // This pattern is automatically generated from aarch64_ad.m4
13387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13388 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13389 %{
13390   match(Set dst (SubI src1 (AndI src2 mask)));
13391   ins_cost(INSN_COST);
13392   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13393 
13394    ins_encode %{
13395      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13396             as_Register($src2$$reg), ext::uxtb);
13397    %}
13398   ins_pipe(ialu_reg_reg);
13399 %}
13400 
13401 // This pattern is automatically generated from aarch64_ad.m4
13402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13403 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13404 %{
13405   match(Set dst (SubI src1 (AndI src2 mask)));
13406   ins_cost(INSN_COST);
13407   format %{ "subw  $dst, $src1, $src2, uxth" %}
13408 
13409    ins_encode %{
13410      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13411             as_Register($src2$$reg), ext::uxth);
13412    %}
13413   ins_pipe(ialu_reg_reg);
13414 %}
13415 
13416 // This pattern is automatically generated from aarch64_ad.m4
13417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13418 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13419 %{
13420   match(Set dst (SubL src1 (AndL src2 mask)));
13421   ins_cost(INSN_COST);
13422   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13423 
13424    ins_encode %{
13425      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13426             as_Register($src2$$reg), ext::uxtb);
13427    %}
13428   ins_pipe(ialu_reg_reg);
13429 %}
13430 
13431 // This pattern is automatically generated from aarch64_ad.m4
13432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13433 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13434 %{
13435   match(Set dst (SubL src1 (AndL src2 mask)));
13436   ins_cost(INSN_COST);
13437   format %{ "sub  $dst, $src1, $src2, uxth" %}
13438 
13439    ins_encode %{
13440      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13441             as_Register($src2$$reg), ext::uxth);
13442    %}
13443   ins_pipe(ialu_reg_reg);
13444 %}
13445 
13446 // This pattern is automatically generated from aarch64_ad.m4
13447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13448 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13449 %{
13450   match(Set dst (SubL src1 (AndL src2 mask)));
13451   ins_cost(INSN_COST);
13452   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13453 
13454    ins_encode %{
13455      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13456             as_Register($src2$$reg), ext::uxtw);
13457    %}
13458   ins_pipe(ialu_reg_reg);
13459 %}
13460 
13461 
13462 // This pattern is automatically generated from aarch64_ad.m4
13463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13464 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13465 %{
13466   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13467   ins_cost(1.9 * INSN_COST);
13468   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13469 
13470    ins_encode %{
13471      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13472             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13473    %}
13474   ins_pipe(ialu_reg_reg_shift);
13475 %}
13476 
13477 // This pattern is automatically generated from aarch64_ad.m4
13478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13479 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13480 %{
13481   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13482   ins_cost(1.9 * INSN_COST);
13483   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13484 
13485    ins_encode %{
13486      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13487             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13488    %}
13489   ins_pipe(ialu_reg_reg_shift);
13490 %}
13491 
13492 // This pattern is automatically generated from aarch64_ad.m4
13493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13494 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13495 %{
13496   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13497   ins_cost(1.9 * INSN_COST);
13498   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13499 
13500    ins_encode %{
13501      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13502             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13503    %}
13504   ins_pipe(ialu_reg_reg_shift);
13505 %}
13506 
13507 // This pattern is automatically generated from aarch64_ad.m4
13508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13509 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13510 %{
13511   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13512   ins_cost(1.9 * INSN_COST);
13513   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13514 
13515    ins_encode %{
13516      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13517             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13518    %}
13519   ins_pipe(ialu_reg_reg_shift);
13520 %}
13521 
13522 // This pattern is automatically generated from aarch64_ad.m4
13523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13524 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13525 %{
13526   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13527   ins_cost(1.9 * INSN_COST);
13528   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13529 
13530    ins_encode %{
13531      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13532             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13533    %}
13534   ins_pipe(ialu_reg_reg_shift);
13535 %}
13536 
13537 // This pattern is automatically generated from aarch64_ad.m4
13538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13539 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13540 %{
13541   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13542   ins_cost(1.9 * INSN_COST);
13543   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13544 
13545    ins_encode %{
13546      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13547             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13548    %}
13549   ins_pipe(ialu_reg_reg_shift);
13550 %}
13551 
13552 // This pattern is automatically generated from aarch64_ad.m4
13553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13554 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13555 %{
13556   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13557   ins_cost(1.9 * INSN_COST);
13558   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13559 
13560    ins_encode %{
13561      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13562             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13563    %}
13564   ins_pipe(ialu_reg_reg_shift);
13565 %}
13566 
13567 // This pattern is automatically generated from aarch64_ad.m4
13568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13569 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13570 %{
13571   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13572   ins_cost(1.9 * INSN_COST);
13573   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13574 
13575    ins_encode %{
13576      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13577             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13578    %}
13579   ins_pipe(ialu_reg_reg_shift);
13580 %}
13581 
13582 // This pattern is automatically generated from aarch64_ad.m4
13583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13584 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13585 %{
13586   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13587   ins_cost(1.9 * INSN_COST);
13588   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13589 
13590    ins_encode %{
13591      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13592             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13593    %}
13594   ins_pipe(ialu_reg_reg_shift);
13595 %}
13596 
13597 // This pattern is automatically generated from aarch64_ad.m4
13598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13599 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13600 %{
13601   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13602   ins_cost(1.9 * INSN_COST);
13603   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13604 
13605    ins_encode %{
13606      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13607             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13608    %}
13609   ins_pipe(ialu_reg_reg_shift);
13610 %}
13611 
13612 // This pattern is automatically generated from aarch64_ad.m4
13613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13614 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13615 %{
13616   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13617   ins_cost(1.9 * INSN_COST);
13618   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13619 
13620    ins_encode %{
13621      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13622             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13623    %}
13624   ins_pipe(ialu_reg_reg_shift);
13625 %}
13626 
13627 // This pattern is automatically generated from aarch64_ad.m4
13628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13629 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13630 %{
13631   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13632   ins_cost(1.9 * INSN_COST);
13633   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13634 
13635    ins_encode %{
13636      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13637             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13638    %}
13639   ins_pipe(ialu_reg_reg_shift);
13640 %}
13641 
13642 // This pattern is automatically generated from aarch64_ad.m4
13643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13644 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13645 %{
13646   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13647   ins_cost(1.9 * INSN_COST);
13648   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13649 
13650    ins_encode %{
13651      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13652             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13653    %}
13654   ins_pipe(ialu_reg_reg_shift);
13655 %}
13656 
13657 // This pattern is automatically generated from aarch64_ad.m4
13658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13659 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13660 %{
13661   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13662   ins_cost(1.9 * INSN_COST);
13663   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13664 
13665    ins_encode %{
13666      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13667             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13668    %}
13669   ins_pipe(ialu_reg_reg_shift);
13670 %}
13671 
13672 // This pattern is automatically generated from aarch64_ad.m4
13673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13674 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13675 %{
13676   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13677   ins_cost(1.9 * INSN_COST);
13678   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13679 
13680    ins_encode %{
13681      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13682             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13683    %}
13684   ins_pipe(ialu_reg_reg_shift);
13685 %}
13686 
13687 // This pattern is automatically generated from aarch64_ad.m4
13688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13689 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13690 %{
13691   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13692   ins_cost(1.9 * INSN_COST);
13693   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13694 
13695    ins_encode %{
13696      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13697             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13698    %}
13699   ins_pipe(ialu_reg_reg_shift);
13700 %}
13701 
13702 // This pattern is automatically generated from aarch64_ad.m4
13703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13704 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13705 %{
13706   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13707   ins_cost(1.9 * INSN_COST);
13708   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13709 
13710    ins_encode %{
13711      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13712             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13713    %}
13714   ins_pipe(ialu_reg_reg_shift);
13715 %}
13716 
13717 // This pattern is automatically generated from aarch64_ad.m4
13718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13719 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13720 %{
13721   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13722   ins_cost(1.9 * INSN_COST);
13723   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13724 
13725    ins_encode %{
13726      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13727             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13728    %}
13729   ins_pipe(ialu_reg_reg_shift);
13730 %}
13731 
13732 // This pattern is automatically generated from aarch64_ad.m4
13733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13734 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13735 %{
13736   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13737   ins_cost(1.9 * INSN_COST);
13738   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13739 
13740    ins_encode %{
13741      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13742             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13743    %}
13744   ins_pipe(ialu_reg_reg_shift);
13745 %}
13746 
13747 // This pattern is automatically generated from aarch64_ad.m4
13748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13749 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13750 %{
13751   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13752   ins_cost(1.9 * INSN_COST);
13753   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13754 
13755    ins_encode %{
13756      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13757             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13758    %}
13759   ins_pipe(ialu_reg_reg_shift);
13760 %}
13761 
13762 // This pattern is automatically generated from aarch64_ad.m4
13763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13764 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13765 %{
13766   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13767   ins_cost(1.9 * INSN_COST);
13768   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13769 
13770    ins_encode %{
13771      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13772             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13773    %}
13774   ins_pipe(ialu_reg_reg_shift);
13775 %}
13776 
13777 // This pattern is automatically generated from aarch64_ad.m4
13778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13779 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13780 %{
13781   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13782   ins_cost(1.9 * INSN_COST);
13783   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13784 
13785    ins_encode %{
13786      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13787             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13788    %}
13789   ins_pipe(ialu_reg_reg_shift);
13790 %}
13791 
13792 
13793 
13794 // END This section of the file is automatically generated. Do not edit --------------
13795 
13796 
13797 // ============================================================================
13798 // Floating Point Arithmetic Instructions
13799 
13800 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13801   match(Set dst (AddF src1 src2));
13802 
13803   ins_cost(INSN_COST * 5);
13804   format %{ "fadds   $dst, $src1, $src2" %}
13805 
13806   ins_encode %{
13807     __ fadds(as_FloatRegister($dst$$reg),
13808              as_FloatRegister($src1$$reg),
13809              as_FloatRegister($src2$$reg));
13810   %}
13811 
13812   ins_pipe(fp_dop_reg_reg_s);
13813 %}
13814 
13815 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13816   match(Set dst (AddD src1 src2));
13817 
13818   ins_cost(INSN_COST * 5);
13819   format %{ "faddd   $dst, $src1, $src2" %}
13820 
13821   ins_encode %{
13822     __ faddd(as_FloatRegister($dst$$reg),
13823              as_FloatRegister($src1$$reg),
13824              as_FloatRegister($src2$$reg));
13825   %}
13826 
13827   ins_pipe(fp_dop_reg_reg_d);
13828 %}
13829 
13830 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13831   match(Set dst (SubF src1 src2));
13832 
13833   ins_cost(INSN_COST * 5);
13834   format %{ "fsubs   $dst, $src1, $src2" %}
13835 
13836   ins_encode %{
13837     __ fsubs(as_FloatRegister($dst$$reg),
13838              as_FloatRegister($src1$$reg),
13839              as_FloatRegister($src2$$reg));
13840   %}
13841 
13842   ins_pipe(fp_dop_reg_reg_s);
13843 %}
13844 
13845 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13846   match(Set dst (SubD src1 src2));
13847 
13848   ins_cost(INSN_COST * 5);
13849   format %{ "fsubd   $dst, $src1, $src2" %}
13850 
13851   ins_encode %{
13852     __ fsubd(as_FloatRegister($dst$$reg),
13853              as_FloatRegister($src1$$reg),
13854              as_FloatRegister($src2$$reg));
13855   %}
13856 
13857   ins_pipe(fp_dop_reg_reg_d);
13858 %}
13859 
13860 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13861   match(Set dst (MulF src1 src2));
13862 
13863   ins_cost(INSN_COST * 6);
13864   format %{ "fmuls   $dst, $src1, $src2" %}
13865 
13866   ins_encode %{
13867     __ fmuls(as_FloatRegister($dst$$reg),
13868              as_FloatRegister($src1$$reg),
13869              as_FloatRegister($src2$$reg));
13870   %}
13871 
13872   ins_pipe(fp_dop_reg_reg_s);
13873 %}
13874 
13875 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13876   match(Set dst (MulD src1 src2));
13877 
13878   ins_cost(INSN_COST * 6);
13879   format %{ "fmuld   $dst, $src1, $src2" %}
13880 
13881   ins_encode %{
13882     __ fmuld(as_FloatRegister($dst$$reg),
13883              as_FloatRegister($src1$$reg),
13884              as_FloatRegister($src2$$reg));
13885   %}
13886 
13887   ins_pipe(fp_dop_reg_reg_d);
13888 %}
13889 
13890 // src1 * src2 + src3
13891 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13892   predicate(UseFMA);
13893   match(Set dst (FmaF src3 (Binary src1 src2)));
13894 
13895   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13896 
13897   ins_encode %{
13898     __ fmadds(as_FloatRegister($dst$$reg),
13899              as_FloatRegister($src1$$reg),
13900              as_FloatRegister($src2$$reg),
13901              as_FloatRegister($src3$$reg));
13902   %}
13903 
13904   ins_pipe(pipe_class_default);
13905 %}
13906 
13907 // src1 * src2 + src3
13908 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13909   predicate(UseFMA);
13910   match(Set dst (FmaD src3 (Binary src1 src2)));
13911 
13912   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13913 
13914   ins_encode %{
13915     __ fmaddd(as_FloatRegister($dst$$reg),
13916              as_FloatRegister($src1$$reg),
13917              as_FloatRegister($src2$$reg),
13918              as_FloatRegister($src3$$reg));
13919   %}
13920 
13921   ins_pipe(pipe_class_default);
13922 %}
13923 
13924 // -src1 * src2 + src3
13925 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13926   predicate(UseFMA);
13927   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13928   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13929 
13930   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13931 
13932   ins_encode %{
13933     __ fmsubs(as_FloatRegister($dst$$reg),
13934               as_FloatRegister($src1$$reg),
13935               as_FloatRegister($src2$$reg),
13936               as_FloatRegister($src3$$reg));
13937   %}
13938 
13939   ins_pipe(pipe_class_default);
13940 %}
13941 
13942 // -src1 * src2 + src3
13943 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13944   predicate(UseFMA);
13945   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13946   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13947 
13948   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13949 
13950   ins_encode %{
13951     __ fmsubd(as_FloatRegister($dst$$reg),
13952               as_FloatRegister($src1$$reg),
13953               as_FloatRegister($src2$$reg),
13954               as_FloatRegister($src3$$reg));
13955   %}
13956 
13957   ins_pipe(pipe_class_default);
13958 %}
13959 
13960 // -src1 * src2 - src3
13961 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13962   predicate(UseFMA);
13963   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13964   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13965 
13966   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13967 
13968   ins_encode %{
13969     __ fnmadds(as_FloatRegister($dst$$reg),
13970                as_FloatRegister($src1$$reg),
13971                as_FloatRegister($src2$$reg),
13972                as_FloatRegister($src3$$reg));
13973   %}
13974 
13975   ins_pipe(pipe_class_default);
13976 %}
13977 
13978 // -src1 * src2 - src3
13979 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13980   predicate(UseFMA);
13981   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13982   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13983 
13984   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13985 
13986   ins_encode %{
13987     __ fnmaddd(as_FloatRegister($dst$$reg),
13988                as_FloatRegister($src1$$reg),
13989                as_FloatRegister($src2$$reg),
13990                as_FloatRegister($src3$$reg));
13991   %}
13992 
13993   ins_pipe(pipe_class_default);
13994 %}
13995 
13996 // src1 * src2 - src3
13997 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13998   predicate(UseFMA);
13999   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14000 
14001   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14002 
14003   ins_encode %{
14004     __ fnmsubs(as_FloatRegister($dst$$reg),
14005                as_FloatRegister($src1$$reg),
14006                as_FloatRegister($src2$$reg),
14007                as_FloatRegister($src3$$reg));
14008   %}
14009 
14010   ins_pipe(pipe_class_default);
14011 %}
14012 
14013 // src1 * src2 - src3
14014 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14015   predicate(UseFMA);
14016   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14017 
14018   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14019 
14020   ins_encode %{
14021   // n.b. insn name should be fnmsubd
14022     __ fnmsub(as_FloatRegister($dst$$reg),
14023               as_FloatRegister($src1$$reg),
14024               as_FloatRegister($src2$$reg),
14025               as_FloatRegister($src3$$reg));
14026   %}
14027 
14028   ins_pipe(pipe_class_default);
14029 %}
14030 
14031 
14032 // Math.max(FF)F
14033 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14034   match(Set dst (MaxF src1 src2));
14035 
14036   format %{ "fmaxs   $dst, $src1, $src2" %}
14037   ins_encode %{
14038     __ fmaxs(as_FloatRegister($dst$$reg),
14039              as_FloatRegister($src1$$reg),
14040              as_FloatRegister($src2$$reg));
14041   %}
14042 
14043   ins_pipe(fp_dop_reg_reg_s);
14044 %}
14045 
14046 // Math.min(FF)F
14047 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14048   match(Set dst (MinF src1 src2));
14049 
14050   format %{ "fmins   $dst, $src1, $src2" %}
14051   ins_encode %{
14052     __ fmins(as_FloatRegister($dst$$reg),
14053              as_FloatRegister($src1$$reg),
14054              as_FloatRegister($src2$$reg));
14055   %}
14056 
14057   ins_pipe(fp_dop_reg_reg_s);
14058 %}
14059 
14060 // Math.max(DD)D
14061 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14062   match(Set dst (MaxD src1 src2));
14063 
14064   format %{ "fmaxd   $dst, $src1, $src2" %}
14065   ins_encode %{
14066     __ fmaxd(as_FloatRegister($dst$$reg),
14067              as_FloatRegister($src1$$reg),
14068              as_FloatRegister($src2$$reg));
14069   %}
14070 
14071   ins_pipe(fp_dop_reg_reg_d);
14072 %}
14073 
14074 // Math.min(DD)D
14075 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14076   match(Set dst (MinD src1 src2));
14077 
14078   format %{ "fmind   $dst, $src1, $src2" %}
14079   ins_encode %{
14080     __ fmind(as_FloatRegister($dst$$reg),
14081              as_FloatRegister($src1$$reg),
14082              as_FloatRegister($src2$$reg));
14083   %}
14084 
14085   ins_pipe(fp_dop_reg_reg_d);
14086 %}
14087 
14088 
14089 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14090   match(Set dst (DivF src1  src2));
14091 
14092   ins_cost(INSN_COST * 18);
14093   format %{ "fdivs   $dst, $src1, $src2" %}
14094 
14095   ins_encode %{
14096     __ fdivs(as_FloatRegister($dst$$reg),
14097              as_FloatRegister($src1$$reg),
14098              as_FloatRegister($src2$$reg));
14099   %}
14100 
14101   ins_pipe(fp_div_s);
14102 %}
14103 
14104 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14105   match(Set dst (DivD src1  src2));
14106 
14107   ins_cost(INSN_COST * 32);
14108   format %{ "fdivd   $dst, $src1, $src2" %}
14109 
14110   ins_encode %{
14111     __ fdivd(as_FloatRegister($dst$$reg),
14112              as_FloatRegister($src1$$reg),
14113              as_FloatRegister($src2$$reg));
14114   %}
14115 
14116   ins_pipe(fp_div_d);
14117 %}
14118 
14119 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14120   match(Set dst (NegF src));
14121 
14122   ins_cost(INSN_COST * 3);
14123   format %{ "fneg   $dst, $src" %}
14124 
14125   ins_encode %{
14126     __ fnegs(as_FloatRegister($dst$$reg),
14127              as_FloatRegister($src$$reg));
14128   %}
14129 
14130   ins_pipe(fp_uop_s);
14131 %}
14132 
14133 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14134   match(Set dst (NegD src));
14135 
14136   ins_cost(INSN_COST * 3);
14137   format %{ "fnegd   $dst, $src" %}
14138 
14139   ins_encode %{
14140     __ fnegd(as_FloatRegister($dst$$reg),
14141              as_FloatRegister($src$$reg));
14142   %}
14143 
14144   ins_pipe(fp_uop_d);
14145 %}
14146 
14147 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14148 %{
14149   match(Set dst (AbsI src));
14150 
14151   effect(KILL cr);
14152   ins_cost(INSN_COST * 2);
14153   format %{ "cmpw  $src, zr\n\t"
14154             "cnegw $dst, $src, Assembler::LT\t# int abs"
14155   %}
14156 
14157   ins_encode %{
14158     __ cmpw(as_Register($src$$reg), zr);
14159     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14160   %}
14161   ins_pipe(pipe_class_default);
14162 %}
14163 
14164 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14165 %{
14166   match(Set dst (AbsL src));
14167 
14168   effect(KILL cr);
14169   ins_cost(INSN_COST * 2);
14170   format %{ "cmp  $src, zr\n\t"
14171             "cneg $dst, $src, Assembler::LT\t# long abs"
14172   %}
14173 
14174   ins_encode %{
14175     __ cmp(as_Register($src$$reg), zr);
14176     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14177   %}
14178   ins_pipe(pipe_class_default);
14179 %}
14180 
14181 instruct absF_reg(vRegF dst, vRegF src) %{
14182   match(Set dst (AbsF src));
14183 
14184   ins_cost(INSN_COST * 3);
14185   format %{ "fabss   $dst, $src" %}
14186   ins_encode %{
14187     __ fabss(as_FloatRegister($dst$$reg),
14188              as_FloatRegister($src$$reg));
14189   %}
14190 
14191   ins_pipe(fp_uop_s);
14192 %}
14193 
14194 instruct absD_reg(vRegD dst, vRegD src) %{
14195   match(Set dst (AbsD src));
14196 
14197   ins_cost(INSN_COST * 3);
14198   format %{ "fabsd   $dst, $src" %}
14199   ins_encode %{
14200     __ fabsd(as_FloatRegister($dst$$reg),
14201              as_FloatRegister($src$$reg));
14202   %}
14203 
14204   ins_pipe(fp_uop_d);
14205 %}
14206 
14207 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14208   match(Set dst (AbsF (SubF src1 src2)));
14209 
14210   ins_cost(INSN_COST * 3);
14211   format %{ "fabds   $dst, $src1, $src2" %}
14212   ins_encode %{
14213     __ fabds(as_FloatRegister($dst$$reg),
14214              as_FloatRegister($src1$$reg),
14215              as_FloatRegister($src2$$reg));
14216   %}
14217 
14218   ins_pipe(fp_uop_s);
14219 %}
14220 
14221 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14222   match(Set dst (AbsD (SubD src1 src2)));
14223 
14224   ins_cost(INSN_COST * 3);
14225   format %{ "fabdd   $dst, $src1, $src2" %}
14226   ins_encode %{
14227     __ fabdd(as_FloatRegister($dst$$reg),
14228              as_FloatRegister($src1$$reg),
14229              as_FloatRegister($src2$$reg));
14230   %}
14231 
14232   ins_pipe(fp_uop_d);
14233 %}
14234 
14235 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14236   match(Set dst (SqrtD src));
14237 
14238   ins_cost(INSN_COST * 50);
14239   format %{ "fsqrtd  $dst, $src" %}
14240   ins_encode %{
14241     __ fsqrtd(as_FloatRegister($dst$$reg),
14242              as_FloatRegister($src$$reg));
14243   %}
14244 
14245   ins_pipe(fp_div_s);
14246 %}
14247 
14248 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14249   match(Set dst (SqrtF src));
14250 
14251   ins_cost(INSN_COST * 50);
14252   format %{ "fsqrts  $dst, $src" %}
14253   ins_encode %{
14254     __ fsqrts(as_FloatRegister($dst$$reg),
14255              as_FloatRegister($src$$reg));
14256   %}
14257 
14258   ins_pipe(fp_div_d);
14259 %}
14260 
14261 // Math.rint, floor, ceil
14262 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14263   match(Set dst (RoundDoubleMode src rmode));
14264   format %{ "frint  $dst, $src, $rmode" %}
14265   ins_encode %{
14266     switch ($rmode$$constant) {
14267       case RoundDoubleModeNode::rmode_rint:
14268         __ frintnd(as_FloatRegister($dst$$reg),
14269                    as_FloatRegister($src$$reg));
14270         break;
14271       case RoundDoubleModeNode::rmode_floor:
14272         __ frintmd(as_FloatRegister($dst$$reg),
14273                    as_FloatRegister($src$$reg));
14274         break;
14275       case RoundDoubleModeNode::rmode_ceil:
14276         __ frintpd(as_FloatRegister($dst$$reg),
14277                    as_FloatRegister($src$$reg));
14278         break;
14279     }
14280   %}
14281   ins_pipe(fp_uop_d);
14282 %}
14283 
14284 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14285   match(Set dst (CopySignD src1 (Binary src2 zero)));
14286   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14287   format %{ "CopySignD  $dst $src1 $src2" %}
14288   ins_encode %{
14289     FloatRegister dst = as_FloatRegister($dst$$reg),
14290                   src1 = as_FloatRegister($src1$$reg),
14291                   src2 = as_FloatRegister($src2$$reg),
14292                   zero = as_FloatRegister($zero$$reg);
14293     __ fnegd(dst, zero);
14294     __ bsl(dst, __ T8B, src2, src1);
14295   %}
14296   ins_pipe(fp_uop_d);
14297 %}
14298 
14299 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14300   match(Set dst (CopySignF src1 src2));
14301   effect(TEMP_DEF dst, USE src1, USE src2);
14302   format %{ "CopySignF  $dst $src1 $src2" %}
14303   ins_encode %{
14304     FloatRegister dst = as_FloatRegister($dst$$reg),
14305                   src1 = as_FloatRegister($src1$$reg),
14306                   src2 = as_FloatRegister($src2$$reg);
14307     __ movi(dst, __ T2S, 0x80, 24);
14308     __ bsl(dst, __ T8B, src2, src1);
14309   %}
14310   ins_pipe(fp_uop_d);
14311 %}
14312 
14313 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14314   match(Set dst (SignumD src (Binary zero one)));
14315   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14316   format %{ "signumD  $dst, $src" %}
14317   ins_encode %{
14318     FloatRegister src = as_FloatRegister($src$$reg),
14319                   dst = as_FloatRegister($dst$$reg),
14320                   zero = as_FloatRegister($zero$$reg),
14321                   one = as_FloatRegister($one$$reg);
14322     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14323     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14324     // Bit selection instruction gets bit from "one" for each enabled bit in
14325     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14326     // NaN the whole "src" will be copied because "dst" is zero. For all other
14327     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14328     // from "src", and all other bits are copied from 1.0.
14329     __ bsl(dst, __ T8B, one, src);
14330   %}
14331   ins_pipe(fp_uop_d);
14332 %}
14333 
14334 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14335   match(Set dst (SignumF src (Binary zero one)));
14336   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14337   format %{ "signumF  $dst, $src" %}
14338   ins_encode %{
14339     FloatRegister src = as_FloatRegister($src$$reg),
14340                   dst = as_FloatRegister($dst$$reg),
14341                   zero = as_FloatRegister($zero$$reg),
14342                   one = as_FloatRegister($one$$reg);
14343     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14344     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14345     // Bit selection instruction gets bit from "one" for each enabled bit in
14346     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14347     // NaN the whole "src" will be copied because "dst" is zero. For all other
14348     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14349     // from "src", and all other bits are copied from 1.0.
14350     __ bsl(dst, __ T8B, one, src);
14351   %}
14352   ins_pipe(fp_uop_d);
14353 %}
14354 
14355 // ============================================================================
14356 // Logical Instructions
14357 
14358 // Integer Logical Instructions
14359 
14360 // And Instructions
14361 
14362 
14363 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14364   match(Set dst (AndI src1 src2));
14365 
14366   format %{ "andw  $dst, $src1, $src2\t# int" %}
14367 
14368   ins_cost(INSN_COST);
14369   ins_encode %{
14370     __ andw(as_Register($dst$$reg),
14371             as_Register($src1$$reg),
14372             as_Register($src2$$reg));
14373   %}
14374 
14375   ins_pipe(ialu_reg_reg);
14376 %}
14377 
14378 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14379   match(Set dst (AndI src1 src2));
14380 
14381   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14382 
14383   ins_cost(INSN_COST);
14384   ins_encode %{
14385     __ andw(as_Register($dst$$reg),
14386             as_Register($src1$$reg),
14387             (uint64_t)($src2$$constant));
14388   %}
14389 
14390   ins_pipe(ialu_reg_imm);
14391 %}
14392 
14393 // Or Instructions
14394 
14395 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14396   match(Set dst (OrI src1 src2));
14397 
14398   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14399 
14400   ins_cost(INSN_COST);
14401   ins_encode %{
14402     __ orrw(as_Register($dst$$reg),
14403             as_Register($src1$$reg),
14404             as_Register($src2$$reg));
14405   %}
14406 
14407   ins_pipe(ialu_reg_reg);
14408 %}
14409 
14410 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14411   match(Set dst (OrI src1 src2));
14412 
14413   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14414 
14415   ins_cost(INSN_COST);
14416   ins_encode %{
14417     __ orrw(as_Register($dst$$reg),
14418             as_Register($src1$$reg),
14419             (uint64_t)($src2$$constant));
14420   %}
14421 
14422   ins_pipe(ialu_reg_imm);
14423 %}
14424 
14425 // Xor Instructions
14426 
14427 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14428   match(Set dst (XorI src1 src2));
14429 
14430   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14431 
14432   ins_cost(INSN_COST);
14433   ins_encode %{
14434     __ eorw(as_Register($dst$$reg),
14435             as_Register($src1$$reg),
14436             as_Register($src2$$reg));
14437   %}
14438 
14439   ins_pipe(ialu_reg_reg);
14440 %}
14441 
14442 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14443   match(Set dst (XorI src1 src2));
14444 
14445   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14446 
14447   ins_cost(INSN_COST);
14448   ins_encode %{
14449     __ eorw(as_Register($dst$$reg),
14450             as_Register($src1$$reg),
14451             (uint64_t)($src2$$constant));
14452   %}
14453 
14454   ins_pipe(ialu_reg_imm);
14455 %}
14456 
14457 // Long Logical Instructions
14458 // TODO
14459 
14460 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14461   match(Set dst (AndL src1 src2));
14462 
14463   format %{ "and  $dst, $src1, $src2\t# int" %}
14464 
14465   ins_cost(INSN_COST);
14466   ins_encode %{
14467     __ andr(as_Register($dst$$reg),
14468             as_Register($src1$$reg),
14469             as_Register($src2$$reg));
14470   %}
14471 
14472   ins_pipe(ialu_reg_reg);
14473 %}
14474 
14475 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14476   match(Set dst (AndL src1 src2));
14477 
14478   format %{ "and  $dst, $src1, $src2\t# int" %}
14479 
14480   ins_cost(INSN_COST);
14481   ins_encode %{
14482     __ andr(as_Register($dst$$reg),
14483             as_Register($src1$$reg),
14484             (uint64_t)($src2$$constant));
14485   %}
14486 
14487   ins_pipe(ialu_reg_imm);
14488 %}
14489 
14490 // Or Instructions
14491 
14492 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14493   match(Set dst (OrL src1 src2));
14494 
14495   format %{ "orr  $dst, $src1, $src2\t# int" %}
14496 
14497   ins_cost(INSN_COST);
14498   ins_encode %{
14499     __ orr(as_Register($dst$$reg),
14500            as_Register($src1$$reg),
14501            as_Register($src2$$reg));
14502   %}
14503 
14504   ins_pipe(ialu_reg_reg);
14505 %}
14506 
14507 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14508   match(Set dst (OrL src1 src2));
14509 
14510   format %{ "orr  $dst, $src1, $src2\t# int" %}
14511 
14512   ins_cost(INSN_COST);
14513   ins_encode %{
14514     __ orr(as_Register($dst$$reg),
14515            as_Register($src1$$reg),
14516            (uint64_t)($src2$$constant));
14517   %}
14518 
14519   ins_pipe(ialu_reg_imm);
14520 %}
14521 
14522 // Xor Instructions
14523 
14524 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14525   match(Set dst (XorL src1 src2));
14526 
14527   format %{ "eor  $dst, $src1, $src2\t# int" %}
14528 
14529   ins_cost(INSN_COST);
14530   ins_encode %{
14531     __ eor(as_Register($dst$$reg),
14532            as_Register($src1$$reg),
14533            as_Register($src2$$reg));
14534   %}
14535 
14536   ins_pipe(ialu_reg_reg);
14537 %}
14538 
14539 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14540   match(Set dst (XorL src1 src2));
14541 
14542   ins_cost(INSN_COST);
14543   format %{ "eor  $dst, $src1, $src2\t# int" %}
14544 
14545   ins_encode %{
14546     __ eor(as_Register($dst$$reg),
14547            as_Register($src1$$reg),
14548            (uint64_t)($src2$$constant));
14549   %}
14550 
14551   ins_pipe(ialu_reg_imm);
14552 %}
14553 
14554 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14555 %{
14556   match(Set dst (ConvI2L src));
14557 
14558   ins_cost(INSN_COST);
14559   format %{ "sxtw  $dst, $src\t# i2l" %}
14560   ins_encode %{
14561     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14562   %}
14563   ins_pipe(ialu_reg_shift);
14564 %}
14565 
14566 // this pattern occurs in bigmath arithmetic
14567 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14568 %{
14569   match(Set dst (AndL (ConvI2L src) mask));
14570 
14571   ins_cost(INSN_COST);
14572   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14573   ins_encode %{
14574     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14575   %}
14576 
14577   ins_pipe(ialu_reg_shift);
14578 %}
14579 
14580 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14581   match(Set dst (ConvL2I src));
14582 
14583   ins_cost(INSN_COST);
14584   format %{ "movw  $dst, $src \t// l2i" %}
14585 
14586   ins_encode %{
14587     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14588   %}
14589 
14590   ins_pipe(ialu_reg);
14591 %}
14592 
14593 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14594 %{
14595   match(Set dst (Conv2B src));
14596   effect(KILL cr);
14597 
14598   format %{
14599     "cmpw $src, zr\n\t"
14600     "cset $dst, ne"
14601   %}
14602 
14603   ins_encode %{
14604     __ cmpw(as_Register($src$$reg), zr);
14605     __ cset(as_Register($dst$$reg), Assembler::NE);
14606   %}
14607 
14608   ins_pipe(ialu_reg);
14609 %}
14610 
14611 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14612 %{
14613   match(Set dst (Conv2B src));
14614   effect(KILL cr);
14615 
14616   format %{
14617     "cmp  $src, zr\n\t"
14618     "cset $dst, ne"
14619   %}
14620 
14621   ins_encode %{
14622     __ cmp(as_Register($src$$reg), zr);
14623     __ cset(as_Register($dst$$reg), Assembler::NE);
14624   %}
14625 
14626   ins_pipe(ialu_reg);
14627 %}
14628 
14629 instruct convD2F_reg(vRegF dst, vRegD src) %{
14630   match(Set dst (ConvD2F src));
14631 
14632   ins_cost(INSN_COST * 5);
14633   format %{ "fcvtd  $dst, $src \t// d2f" %}
14634 
14635   ins_encode %{
14636     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14637   %}
14638 
14639   ins_pipe(fp_d2f);
14640 %}
14641 
14642 instruct convF2D_reg(vRegD dst, vRegF src) %{
14643   match(Set dst (ConvF2D src));
14644 
14645   ins_cost(INSN_COST * 5);
14646   format %{ "fcvts  $dst, $src \t// f2d" %}
14647 
14648   ins_encode %{
14649     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14650   %}
14651 
14652   ins_pipe(fp_f2d);
14653 %}
14654 
14655 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14656   match(Set dst (ConvF2I src));
14657 
14658   ins_cost(INSN_COST * 5);
14659   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14660 
14661   ins_encode %{
14662     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14663   %}
14664 
14665   ins_pipe(fp_f2i);
14666 %}
14667 
14668 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14669   match(Set dst (ConvF2L src));
14670 
14671   ins_cost(INSN_COST * 5);
14672   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14673 
14674   ins_encode %{
14675     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14676   %}
14677 
14678   ins_pipe(fp_f2l);
14679 %}
14680 
14681 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14682   match(Set dst (ConvI2F src));
14683 
14684   ins_cost(INSN_COST * 5);
14685   format %{ "scvtfws  $dst, $src \t// i2f" %}
14686 
14687   ins_encode %{
14688     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14689   %}
14690 
14691   ins_pipe(fp_i2f);
14692 %}
14693 
14694 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14695   match(Set dst (ConvL2F src));
14696 
14697   ins_cost(INSN_COST * 5);
14698   format %{ "scvtfs  $dst, $src \t// l2f" %}
14699 
14700   ins_encode %{
14701     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14702   %}
14703 
14704   ins_pipe(fp_l2f);
14705 %}
14706 
14707 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14708   match(Set dst (ConvD2I src));
14709 
14710   ins_cost(INSN_COST * 5);
14711   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14712 
14713   ins_encode %{
14714     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14715   %}
14716 
14717   ins_pipe(fp_d2i);
14718 %}
14719 
14720 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14721   match(Set dst (ConvD2L src));
14722 
14723   ins_cost(INSN_COST * 5);
14724   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14725 
14726   ins_encode %{
14727     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14728   %}
14729 
14730   ins_pipe(fp_d2l);
14731 %}
14732 
14733 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14734   match(Set dst (ConvI2D src));
14735 
14736   ins_cost(INSN_COST * 5);
14737   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14738 
14739   ins_encode %{
14740     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14741   %}
14742 
14743   ins_pipe(fp_i2d);
14744 %}
14745 
14746 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14747   match(Set dst (ConvL2D src));
14748 
14749   ins_cost(INSN_COST * 5);
14750   format %{ "scvtfd  $dst, $src \t// l2d" %}
14751 
14752   ins_encode %{
14753     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14754   %}
14755 
14756   ins_pipe(fp_l2d);
14757 %}
14758 
14759 // stack <-> reg and reg <-> reg shuffles with no conversion
14760 
14761 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14762 
14763   match(Set dst (MoveF2I src));
14764 
14765   effect(DEF dst, USE src);
14766 
14767   ins_cost(4 * INSN_COST);
14768 
14769   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14770 
14771   ins_encode %{
14772     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14773   %}
14774 
14775   ins_pipe(iload_reg_reg);
14776 
14777 %}
14778 
14779 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14780 
14781   match(Set dst (MoveI2F src));
14782 
14783   effect(DEF dst, USE src);
14784 
14785   ins_cost(4 * INSN_COST);
14786 
14787   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14788 
14789   ins_encode %{
14790     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14791   %}
14792 
14793   ins_pipe(pipe_class_memory);
14794 
14795 %}
14796 
14797 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14798 
14799   match(Set dst (MoveD2L src));
14800 
14801   effect(DEF dst, USE src);
14802 
14803   ins_cost(4 * INSN_COST);
14804 
14805   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14806 
14807   ins_encode %{
14808     __ ldr($dst$$Register, Address(sp, $src$$disp));
14809   %}
14810 
14811   ins_pipe(iload_reg_reg);
14812 
14813 %}
14814 
14815 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14816 
14817   match(Set dst (MoveL2D src));
14818 
14819   effect(DEF dst, USE src);
14820 
14821   ins_cost(4 * INSN_COST);
14822 
14823   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14824 
14825   ins_encode %{
14826     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14827   %}
14828 
14829   ins_pipe(pipe_class_memory);
14830 
14831 %}
14832 
14833 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14834 
14835   match(Set dst (MoveF2I src));
14836 
14837   effect(DEF dst, USE src);
14838 
14839   ins_cost(INSN_COST);
14840 
14841   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14842 
14843   ins_encode %{
14844     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14845   %}
14846 
14847   ins_pipe(pipe_class_memory);
14848 
14849 %}
14850 
14851 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14852 
14853   match(Set dst (MoveI2F src));
14854 
14855   effect(DEF dst, USE src);
14856 
14857   ins_cost(INSN_COST);
14858 
14859   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14860 
14861   ins_encode %{
14862     __ strw($src$$Register, Address(sp, $dst$$disp));
14863   %}
14864 
14865   ins_pipe(istore_reg_reg);
14866 
14867 %}
14868 
14869 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14870 
14871   match(Set dst (MoveD2L src));
14872 
14873   effect(DEF dst, USE src);
14874 
14875   ins_cost(INSN_COST);
14876 
14877   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14878 
14879   ins_encode %{
14880     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14881   %}
14882 
14883   ins_pipe(pipe_class_memory);
14884 
14885 %}
14886 
14887 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14888 
14889   match(Set dst (MoveL2D src));
14890 
14891   effect(DEF dst, USE src);
14892 
14893   ins_cost(INSN_COST);
14894 
14895   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14896 
14897   ins_encode %{
14898     __ str($src$$Register, Address(sp, $dst$$disp));
14899   %}
14900 
14901   ins_pipe(istore_reg_reg);
14902 
14903 %}
14904 
14905 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14906 
14907   match(Set dst (MoveF2I src));
14908 
14909   effect(DEF dst, USE src);
14910 
14911   ins_cost(INSN_COST);
14912 
14913   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14914 
14915   ins_encode %{
14916     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14917   %}
14918 
14919   ins_pipe(fp_f2i);
14920 
14921 %}
14922 
14923 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14924 
14925   match(Set dst (MoveI2F src));
14926 
14927   effect(DEF dst, USE src);
14928 
14929   ins_cost(INSN_COST);
14930 
14931   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14932 
14933   ins_encode %{
14934     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14935   %}
14936 
14937   ins_pipe(fp_i2f);
14938 
14939 %}
14940 
14941 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14942 
14943   match(Set dst (MoveD2L src));
14944 
14945   effect(DEF dst, USE src);
14946 
14947   ins_cost(INSN_COST);
14948 
14949   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14950 
14951   ins_encode %{
14952     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14953   %}
14954 
14955   ins_pipe(fp_d2l);
14956 
14957 %}
14958 
14959 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14960 
14961   match(Set dst (MoveL2D src));
14962 
14963   effect(DEF dst, USE src);
14964 
14965   ins_cost(INSN_COST);
14966 
14967   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14968 
14969   ins_encode %{
14970     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14971   %}
14972 
14973   ins_pipe(fp_l2d);
14974 
14975 %}
14976 
14977 // ============================================================================
14978 // clearing of an array
14979 
14980 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14981 %{
14982   match(Set dummy (ClearArray cnt base));
14983   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14984 
14985   ins_cost(4 * INSN_COST);
14986   format %{ "ClearArray $cnt, $base" %}
14987 
14988   ins_encode %{
14989     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14990     if (tpc == NULL) {
14991       ciEnv::current()->record_failure("CodeCache is full");
14992       return;
14993     }
14994   %}
14995 
14996   ins_pipe(pipe_class_memory);
14997 %}
14998 
14999 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15000 %{
15001   predicate((uint64_t)n->in(2)->get_long()
15002             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15003   match(Set dummy (ClearArray cnt base));
15004   effect(TEMP temp, USE_KILL base, KILL cr);
15005 
15006   ins_cost(4 * INSN_COST);
15007   format %{ "ClearArray $cnt, $base" %}
15008 
15009   ins_encode %{
15010     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15011   %}
15012 
15013   ins_pipe(pipe_class_memory);
15014 %}
15015 
15016 // ============================================================================
15017 // Overflow Math Instructions
15018 
15019 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15020 %{
15021   match(Set cr (OverflowAddI op1 op2));
15022 
15023   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15024   ins_cost(INSN_COST);
15025   ins_encode %{
15026     __ cmnw($op1$$Register, $op2$$Register);
15027   %}
15028 
15029   ins_pipe(icmp_reg_reg);
15030 %}
15031 
15032 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15033 %{
15034   match(Set cr (OverflowAddI op1 op2));
15035 
15036   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15037   ins_cost(INSN_COST);
15038   ins_encode %{
15039     __ cmnw($op1$$Register, $op2$$constant);
15040   %}
15041 
15042   ins_pipe(icmp_reg_imm);
15043 %}
15044 
15045 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15046 %{
15047   match(Set cr (OverflowAddL op1 op2));
15048 
15049   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15050   ins_cost(INSN_COST);
15051   ins_encode %{
15052     __ cmn($op1$$Register, $op2$$Register);
15053   %}
15054 
15055   ins_pipe(icmp_reg_reg);
15056 %}
15057 
15058 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15059 %{
15060   match(Set cr (OverflowAddL op1 op2));
15061 
15062   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15063   ins_cost(INSN_COST);
15064   ins_encode %{
15065     __ cmn($op1$$Register, $op2$$constant);
15066   %}
15067 
15068   ins_pipe(icmp_reg_imm);
15069 %}
15070 
15071 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15072 %{
15073   match(Set cr (OverflowSubI op1 op2));
15074 
15075   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15076   ins_cost(INSN_COST);
15077   ins_encode %{
15078     __ cmpw($op1$$Register, $op2$$Register);
15079   %}
15080 
15081   ins_pipe(icmp_reg_reg);
15082 %}
15083 
15084 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15085 %{
15086   match(Set cr (OverflowSubI op1 op2));
15087 
15088   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15089   ins_cost(INSN_COST);
15090   ins_encode %{
15091     __ cmpw($op1$$Register, $op2$$constant);
15092   %}
15093 
15094   ins_pipe(icmp_reg_imm);
15095 %}
15096 
15097 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15098 %{
15099   match(Set cr (OverflowSubL op1 op2));
15100 
15101   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15102   ins_cost(INSN_COST);
15103   ins_encode %{
15104     __ cmp($op1$$Register, $op2$$Register);
15105   %}
15106 
15107   ins_pipe(icmp_reg_reg);
15108 %}
15109 
15110 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15111 %{
15112   match(Set cr (OverflowSubL op1 op2));
15113 
15114   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15115   ins_cost(INSN_COST);
15116   ins_encode %{
15117     __ subs(zr, $op1$$Register, $op2$$constant);
15118   %}
15119 
15120   ins_pipe(icmp_reg_imm);
15121 %}
15122 
15123 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15124 %{
15125   match(Set cr (OverflowSubI zero op1));
15126 
15127   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15128   ins_cost(INSN_COST);
15129   ins_encode %{
15130     __ cmpw(zr, $op1$$Register);
15131   %}
15132 
15133   ins_pipe(icmp_reg_imm);
15134 %}
15135 
15136 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15137 %{
15138   match(Set cr (OverflowSubL zero op1));
15139 
15140   format %{ "cmp   zr, $op1\t# overflow check long" %}
15141   ins_cost(INSN_COST);
15142   ins_encode %{
15143     __ cmp(zr, $op1$$Register);
15144   %}
15145 
15146   ins_pipe(icmp_reg_imm);
15147 %}
15148 
15149 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15150 %{
15151   match(Set cr (OverflowMulI op1 op2));
15152 
15153   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15154             "cmp   rscratch1, rscratch1, sxtw\n\t"
15155             "movw  rscratch1, #0x80000000\n\t"
15156             "cselw rscratch1, rscratch1, zr, NE\n\t"
15157             "cmpw  rscratch1, #1" %}
15158   ins_cost(5 * INSN_COST);
15159   ins_encode %{
15160     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15161     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15162     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15163     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15164     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15165   %}
15166 
15167   ins_pipe(pipe_slow);
15168 %}
15169 
15170 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15171 %{
15172   match(If cmp (OverflowMulI op1 op2));
15173   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15174             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15175   effect(USE labl, KILL cr);
15176 
15177   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15178             "cmp   rscratch1, rscratch1, sxtw\n\t"
15179             "b$cmp   $labl" %}
15180   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15181   ins_encode %{
15182     Label* L = $labl$$label;
15183     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15184     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15185     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15186     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15187   %}
15188 
15189   ins_pipe(pipe_serial);
15190 %}
15191 
15192 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15193 %{
15194   match(Set cr (OverflowMulL op1 op2));
15195 
15196   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15197             "smulh rscratch2, $op1, $op2\n\t"
15198             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15199             "movw  rscratch1, #0x80000000\n\t"
15200             "cselw rscratch1, rscratch1, zr, NE\n\t"
15201             "cmpw  rscratch1, #1" %}
15202   ins_cost(6 * INSN_COST);
15203   ins_encode %{
15204     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15205     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15206     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15207     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15208     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15209     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15210   %}
15211 
15212   ins_pipe(pipe_slow);
15213 %}
15214 
15215 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15216 %{
15217   match(If cmp (OverflowMulL op1 op2));
15218   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15219             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15220   effect(USE labl, KILL cr);
15221 
15222   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15223             "smulh rscratch2, $op1, $op2\n\t"
15224             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15225             "b$cmp $labl" %}
15226   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15227   ins_encode %{
15228     Label* L = $labl$$label;
15229     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15230     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15231     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15232     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15233     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15234   %}
15235 
15236   ins_pipe(pipe_serial);
15237 %}
15238 
15239 // ============================================================================
15240 // Compare Instructions
15241 
15242 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15243 %{
15244   match(Set cr (CmpI op1 op2));
15245 
15246   effect(DEF cr, USE op1, USE op2);
15247 
15248   ins_cost(INSN_COST);
15249   format %{ "cmpw  $op1, $op2" %}
15250 
15251   ins_encode(aarch64_enc_cmpw(op1, op2));
15252 
15253   ins_pipe(icmp_reg_reg);
15254 %}
15255 
15256 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15257 %{
15258   match(Set cr (CmpI op1 zero));
15259 
15260   effect(DEF cr, USE op1);
15261 
15262   ins_cost(INSN_COST);
15263   format %{ "cmpw $op1, 0" %}
15264 
15265   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15266 
15267   ins_pipe(icmp_reg_imm);
15268 %}
15269 
15270 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15271 %{
15272   match(Set cr (CmpI op1 op2));
15273 
15274   effect(DEF cr, USE op1);
15275 
15276   ins_cost(INSN_COST);
15277   format %{ "cmpw  $op1, $op2" %}
15278 
15279   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15280 
15281   ins_pipe(icmp_reg_imm);
15282 %}
15283 
15284 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15285 %{
15286   match(Set cr (CmpI op1 op2));
15287 
15288   effect(DEF cr, USE op1);
15289 
15290   ins_cost(INSN_COST * 2);
15291   format %{ "cmpw  $op1, $op2" %}
15292 
15293   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15294 
15295   ins_pipe(icmp_reg_imm);
15296 %}
15297 
15298 // Unsigned compare Instructions; really, same as signed compare
15299 // except it should only be used to feed an If or a CMovI which takes a
15300 // cmpOpU.
15301 
15302 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15303 %{
15304   match(Set cr (CmpU op1 op2));
15305 
15306   effect(DEF cr, USE op1, USE op2);
15307 
15308   ins_cost(INSN_COST);
15309   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15310 
15311   ins_encode(aarch64_enc_cmpw(op1, op2));
15312 
15313   ins_pipe(icmp_reg_reg);
15314 %}
15315 
15316 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15317 %{
15318   match(Set cr (CmpU op1 zero));
15319 
15320   effect(DEF cr, USE op1);
15321 
15322   ins_cost(INSN_COST);
15323   format %{ "cmpw $op1, #0\t# unsigned" %}
15324 
15325   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15326 
15327   ins_pipe(icmp_reg_imm);
15328 %}
15329 
15330 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15331 %{
15332   match(Set cr (CmpU op1 op2));
15333 
15334   effect(DEF cr, USE op1);
15335 
15336   ins_cost(INSN_COST);
15337   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15338 
15339   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15340 
15341   ins_pipe(icmp_reg_imm);
15342 %}
15343 
15344 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15345 %{
15346   match(Set cr (CmpU op1 op2));
15347 
15348   effect(DEF cr, USE op1);
15349 
15350   ins_cost(INSN_COST * 2);
15351   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15352 
15353   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15354 
15355   ins_pipe(icmp_reg_imm);
15356 %}
15357 
15358 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15359 %{
15360   match(Set cr (CmpL op1 op2));
15361 
15362   effect(DEF cr, USE op1, USE op2);
15363 
15364   ins_cost(INSN_COST);
15365   format %{ "cmp  $op1, $op2" %}
15366 
15367   ins_encode(aarch64_enc_cmp(op1, op2));
15368 
15369   ins_pipe(icmp_reg_reg);
15370 %}
15371 
15372 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15373 %{
15374   match(Set cr (CmpL op1 zero));
15375 
15376   effect(DEF cr, USE op1);
15377 
15378   ins_cost(INSN_COST);
15379   format %{ "tst  $op1" %}
15380 
15381   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15382 
15383   ins_pipe(icmp_reg_imm);
15384 %}
15385 
15386 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15387 %{
15388   match(Set cr (CmpL op1 op2));
15389 
15390   effect(DEF cr, USE op1);
15391 
15392   ins_cost(INSN_COST);
15393   format %{ "cmp  $op1, $op2" %}
15394 
15395   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15396 
15397   ins_pipe(icmp_reg_imm);
15398 %}
15399 
15400 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15401 %{
15402   match(Set cr (CmpL op1 op2));
15403 
15404   effect(DEF cr, USE op1);
15405 
15406   ins_cost(INSN_COST * 2);
15407   format %{ "cmp  $op1, $op2" %}
15408 
15409   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15410 
15411   ins_pipe(icmp_reg_imm);
15412 %}
15413 
15414 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15415 %{
15416   match(Set cr (CmpUL op1 op2));
15417 
15418   effect(DEF cr, USE op1, USE op2);
15419 
15420   ins_cost(INSN_COST);
15421   format %{ "cmp  $op1, $op2" %}
15422 
15423   ins_encode(aarch64_enc_cmp(op1, op2));
15424 
15425   ins_pipe(icmp_reg_reg);
15426 %}
15427 
15428 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15429 %{
15430   match(Set cr (CmpUL op1 zero));
15431 
15432   effect(DEF cr, USE op1);
15433 
15434   ins_cost(INSN_COST);
15435   format %{ "tst  $op1" %}
15436 
15437   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15438 
15439   ins_pipe(icmp_reg_imm);
15440 %}
15441 
15442 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15443 %{
15444   match(Set cr (CmpUL op1 op2));
15445 
15446   effect(DEF cr, USE op1);
15447 
15448   ins_cost(INSN_COST);
15449   format %{ "cmp  $op1, $op2" %}
15450 
15451   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15452 
15453   ins_pipe(icmp_reg_imm);
15454 %}
15455 
15456 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15457 %{
15458   match(Set cr (CmpUL op1 op2));
15459 
15460   effect(DEF cr, USE op1);
15461 
15462   ins_cost(INSN_COST * 2);
15463   format %{ "cmp  $op1, $op2" %}
15464 
15465   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15466 
15467   ins_pipe(icmp_reg_imm);
15468 %}
15469 
15470 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15471 %{
15472   match(Set cr (CmpP op1 op2));
15473 
15474   effect(DEF cr, USE op1, USE op2);
15475 
15476   ins_cost(INSN_COST);
15477   format %{ "cmp  $op1, $op2\t // ptr" %}
15478 
15479   ins_encode(aarch64_enc_cmpp(op1, op2));
15480 
15481   ins_pipe(icmp_reg_reg);
15482 %}
15483 
15484 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15485 %{
15486   match(Set cr (CmpN op1 op2));
15487 
15488   effect(DEF cr, USE op1, USE op2);
15489 
15490   ins_cost(INSN_COST);
15491   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15492 
15493   ins_encode(aarch64_enc_cmpn(op1, op2));
15494 
15495   ins_pipe(icmp_reg_reg);
15496 %}
15497 
15498 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15499 %{
15500   match(Set cr (CmpP op1 zero));
15501 
15502   effect(DEF cr, USE op1, USE zero);
15503 
15504   ins_cost(INSN_COST);
15505   format %{ "cmp  $op1, 0\t // ptr" %}
15506 
15507   ins_encode(aarch64_enc_testp(op1));
15508 
15509   ins_pipe(icmp_reg_imm);
15510 %}
15511 
15512 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15513 %{
15514   match(Set cr (CmpN op1 zero));
15515 
15516   effect(DEF cr, USE op1, USE zero);
15517 
15518   ins_cost(INSN_COST);
15519   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15520 
15521   ins_encode(aarch64_enc_testn(op1));
15522 
15523   ins_pipe(icmp_reg_imm);
15524 %}
15525 
15526 // FP comparisons
15527 //
15528 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15529 // using normal cmpOp. See declaration of rFlagsReg for details.
15530 
15531 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15532 %{
15533   match(Set cr (CmpF src1 src2));
15534 
15535   ins_cost(3 * INSN_COST);
15536   format %{ "fcmps $src1, $src2" %}
15537 
15538   ins_encode %{
15539     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15540   %}
15541 
15542   ins_pipe(pipe_class_compare);
15543 %}
15544 
15545 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15546 %{
15547   match(Set cr (CmpF src1 src2));
15548 
15549   ins_cost(3 * INSN_COST);
15550   format %{ "fcmps $src1, 0.0" %}
15551 
15552   ins_encode %{
15553     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15554   %}
15555 
15556   ins_pipe(pipe_class_compare);
15557 %}
15558 // FROM HERE
15559 
15560 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15561 %{
15562   match(Set cr (CmpD src1 src2));
15563 
15564   ins_cost(3 * INSN_COST);
15565   format %{ "fcmpd $src1, $src2" %}
15566 
15567   ins_encode %{
15568     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15569   %}
15570 
15571   ins_pipe(pipe_class_compare);
15572 %}
15573 
15574 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15575 %{
15576   match(Set cr (CmpD src1 src2));
15577 
15578   ins_cost(3 * INSN_COST);
15579   format %{ "fcmpd $src1, 0.0" %}
15580 
15581   ins_encode %{
15582     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15583   %}
15584 
15585   ins_pipe(pipe_class_compare);
15586 %}
15587 
15588 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15589 %{
15590   match(Set dst (CmpF3 src1 src2));
15591   effect(KILL cr);
15592 
15593   ins_cost(5 * INSN_COST);
15594   format %{ "fcmps $src1, $src2\n\t"
15595             "csinvw($dst, zr, zr, eq\n\t"
15596             "csnegw($dst, $dst, $dst, lt)"
15597   %}
15598 
15599   ins_encode %{
15600     Label done;
15601     FloatRegister s1 = as_FloatRegister($src1$$reg);
15602     FloatRegister s2 = as_FloatRegister($src2$$reg);
15603     Register d = as_Register($dst$$reg);
15604     __ fcmps(s1, s2);
15605     // installs 0 if EQ else -1
15606     __ csinvw(d, zr, zr, Assembler::EQ);
15607     // keeps -1 if less or unordered else installs 1
15608     __ csnegw(d, d, d, Assembler::LT);
15609     __ bind(done);
15610   %}
15611 
15612   ins_pipe(pipe_class_default);
15613 
15614 %}
15615 
15616 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15617 %{
15618   match(Set dst (CmpD3 src1 src2));
15619   effect(KILL cr);
15620 
15621   ins_cost(5 * INSN_COST);
15622   format %{ "fcmpd $src1, $src2\n\t"
15623             "csinvw($dst, zr, zr, eq\n\t"
15624             "csnegw($dst, $dst, $dst, lt)"
15625   %}
15626 
15627   ins_encode %{
15628     Label done;
15629     FloatRegister s1 = as_FloatRegister($src1$$reg);
15630     FloatRegister s2 = as_FloatRegister($src2$$reg);
15631     Register d = as_Register($dst$$reg);
15632     __ fcmpd(s1, s2);
15633     // installs 0 if EQ else -1
15634     __ csinvw(d, zr, zr, Assembler::EQ);
15635     // keeps -1 if less or unordered else installs 1
15636     __ csnegw(d, d, d, Assembler::LT);
15637     __ bind(done);
15638   %}
15639   ins_pipe(pipe_class_default);
15640 
15641 %}
15642 
15643 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15644 %{
15645   match(Set dst (CmpF3 src1 zero));
15646   effect(KILL cr);
15647 
15648   ins_cost(5 * INSN_COST);
15649   format %{ "fcmps $src1, 0.0\n\t"
15650             "csinvw($dst, zr, zr, eq\n\t"
15651             "csnegw($dst, $dst, $dst, lt)"
15652   %}
15653 
15654   ins_encode %{
15655     Label done;
15656     FloatRegister s1 = as_FloatRegister($src1$$reg);
15657     Register d = as_Register($dst$$reg);
15658     __ fcmps(s1, 0.0);
15659     // installs 0 if EQ else -1
15660     __ csinvw(d, zr, zr, Assembler::EQ);
15661     // keeps -1 if less or unordered else installs 1
15662     __ csnegw(d, d, d, Assembler::LT);
15663     __ bind(done);
15664   %}
15665 
15666   ins_pipe(pipe_class_default);
15667 
15668 %}
15669 
15670 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15671 %{
15672   match(Set dst (CmpD3 src1 zero));
15673   effect(KILL cr);
15674 
15675   ins_cost(5 * INSN_COST);
15676   format %{ "fcmpd $src1, 0.0\n\t"
15677             "csinvw($dst, zr, zr, eq\n\t"
15678             "csnegw($dst, $dst, $dst, lt)"
15679   %}
15680 
15681   ins_encode %{
15682     Label done;
15683     FloatRegister s1 = as_FloatRegister($src1$$reg);
15684     Register d = as_Register($dst$$reg);
15685     __ fcmpd(s1, 0.0);
15686     // installs 0 if EQ else -1
15687     __ csinvw(d, zr, zr, Assembler::EQ);
15688     // keeps -1 if less or unordered else installs 1
15689     __ csnegw(d, d, d, Assembler::LT);
15690     __ bind(done);
15691   %}
15692   ins_pipe(pipe_class_default);
15693 
15694 %}
15695 
15696 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15697 %{
15698   match(Set dst (CmpLTMask p q));
15699   effect(KILL cr);
15700 
15701   ins_cost(3 * INSN_COST);
15702 
15703   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15704             "csetw $dst, lt\n\t"
15705             "subw $dst, zr, $dst"
15706   %}
15707 
15708   ins_encode %{
15709     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15710     __ csetw(as_Register($dst$$reg), Assembler::LT);
15711     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15712   %}
15713 
15714   ins_pipe(ialu_reg_reg);
15715 %}
15716 
15717 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15718 %{
15719   match(Set dst (CmpLTMask src zero));
15720   effect(KILL cr);
15721 
15722   ins_cost(INSN_COST);
15723 
15724   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15725 
15726   ins_encode %{
15727     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15728   %}
15729 
15730   ins_pipe(ialu_reg_shift);
15731 %}
15732 
15733 // ============================================================================
15734 // Max and Min
15735 
15736 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15737 %{
15738   effect( DEF dst, USE src1, USE src2, USE cr );
15739 
15740   ins_cost(INSN_COST * 2);
15741   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15742 
15743   ins_encode %{
15744     __ cselw(as_Register($dst$$reg),
15745              as_Register($src1$$reg),
15746              as_Register($src2$$reg),
15747              Assembler::LT);
15748   %}
15749 
15750   ins_pipe(icond_reg_reg);
15751 %}
15752 
15753 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15754 %{
15755   match(Set dst (MinI src1 src2));
15756   ins_cost(INSN_COST * 3);
15757 
15758   expand %{
15759     rFlagsReg cr;
15760     compI_reg_reg(cr, src1, src2);
15761     cmovI_reg_reg_lt(dst, src1, src2, cr);
15762   %}
15763 
15764 %}
15765 // FROM HERE
15766 
15767 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15768 %{
15769   effect( DEF dst, USE src1, USE src2, USE cr );
15770 
15771   ins_cost(INSN_COST * 2);
15772   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15773 
15774   ins_encode %{
15775     __ cselw(as_Register($dst$$reg),
15776              as_Register($src1$$reg),
15777              as_Register($src2$$reg),
15778              Assembler::GT);
15779   %}
15780 
15781   ins_pipe(icond_reg_reg);
15782 %}
15783 
15784 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15785 %{
15786   match(Set dst (MaxI src1 src2));
15787   ins_cost(INSN_COST * 3);
15788   expand %{
15789     rFlagsReg cr;
15790     compI_reg_reg(cr, src1, src2);
15791     cmovI_reg_reg_gt(dst, src1, src2, cr);
15792   %}
15793 %}
15794 
15795 // ============================================================================
15796 // Branch Instructions
15797 
15798 // Direct Branch.
15799 instruct branch(label lbl)
15800 %{
15801   match(Goto);
15802 
15803   effect(USE lbl);
15804 
15805   ins_cost(BRANCH_COST);
15806   format %{ "b  $lbl" %}
15807 
15808   ins_encode(aarch64_enc_b(lbl));
15809 
15810   ins_pipe(pipe_branch);
15811 %}
15812 
15813 // Conditional Near Branch
15814 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15815 %{
15816   // Same match rule as `branchConFar'.
15817   match(If cmp cr);
15818 
15819   effect(USE lbl);
15820 
15821   ins_cost(BRANCH_COST);
15822   // If set to 1 this indicates that the current instruction is a
15823   // short variant of a long branch. This avoids using this
15824   // instruction in first-pass matching. It will then only be used in
15825   // the `Shorten_branches' pass.
15826   // ins_short_branch(1);
15827   format %{ "b$cmp  $lbl" %}
15828 
15829   ins_encode(aarch64_enc_br_con(cmp, lbl));
15830 
15831   ins_pipe(pipe_branch_cond);
15832 %}
15833 
15834 // Conditional Near Branch Unsigned
15835 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15836 %{
15837   // Same match rule as `branchConFar'.
15838   match(If cmp cr);
15839 
15840   effect(USE lbl);
15841 
15842   ins_cost(BRANCH_COST);
15843   // If set to 1 this indicates that the current instruction is a
15844   // short variant of a long branch. This avoids using this
15845   // instruction in first-pass matching. It will then only be used in
15846   // the `Shorten_branches' pass.
15847   // ins_short_branch(1);
15848   format %{ "b$cmp  $lbl\t# unsigned" %}
15849 
15850   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15851 
15852   ins_pipe(pipe_branch_cond);
15853 %}
15854 
15855 // Make use of CBZ and CBNZ.  These instructions, as well as being
15856 // shorter than (cmp; branch), have the additional benefit of not
15857 // killing the flags.
15858 
15859 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15860   match(If cmp (CmpI op1 op2));
15861   effect(USE labl);
15862 
15863   ins_cost(BRANCH_COST);
15864   format %{ "cbw$cmp   $op1, $labl" %}
15865   ins_encode %{
15866     Label* L = $labl$$label;
15867     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15868     if (cond == Assembler::EQ)
15869       __ cbzw($op1$$Register, *L);
15870     else
15871       __ cbnzw($op1$$Register, *L);
15872   %}
15873   ins_pipe(pipe_cmp_branch);
15874 %}
15875 
15876 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15877   match(If cmp (CmpL op1 op2));
15878   effect(USE labl);
15879 
15880   ins_cost(BRANCH_COST);
15881   format %{ "cb$cmp   $op1, $labl" %}
15882   ins_encode %{
15883     Label* L = $labl$$label;
15884     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15885     if (cond == Assembler::EQ)
15886       __ cbz($op1$$Register, *L);
15887     else
15888       __ cbnz($op1$$Register, *L);
15889   %}
15890   ins_pipe(pipe_cmp_branch);
15891 %}
15892 
15893 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15894   match(If cmp (CmpP op1 op2));
15895   effect(USE labl);
15896 
15897   ins_cost(BRANCH_COST);
15898   format %{ "cb$cmp   $op1, $labl" %}
15899   ins_encode %{
15900     Label* L = $labl$$label;
15901     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15902     if (cond == Assembler::EQ)
15903       __ cbz($op1$$Register, *L);
15904     else
15905       __ cbnz($op1$$Register, *L);
15906   %}
15907   ins_pipe(pipe_cmp_branch);
15908 %}
15909 
15910 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15911   match(If cmp (CmpN op1 op2));
15912   effect(USE labl);
15913 
15914   ins_cost(BRANCH_COST);
15915   format %{ "cbw$cmp   $op1, $labl" %}
15916   ins_encode %{
15917     Label* L = $labl$$label;
15918     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15919     if (cond == Assembler::EQ)
15920       __ cbzw($op1$$Register, *L);
15921     else
15922       __ cbnzw($op1$$Register, *L);
15923   %}
15924   ins_pipe(pipe_cmp_branch);
15925 %}
15926 
15927 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15928   match(If cmp (CmpP (DecodeN oop) zero));
15929   effect(USE labl);
15930 
15931   ins_cost(BRANCH_COST);
15932   format %{ "cb$cmp   $oop, $labl" %}
15933   ins_encode %{
15934     Label* L = $labl$$label;
15935     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15936     if (cond == Assembler::EQ)
15937       __ cbzw($oop$$Register, *L);
15938     else
15939       __ cbnzw($oop$$Register, *L);
15940   %}
15941   ins_pipe(pipe_cmp_branch);
15942 %}
15943 
15944 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15945   match(If cmp (CmpU op1 op2));
15946   effect(USE labl);
15947 
15948   ins_cost(BRANCH_COST);
15949   format %{ "cbw$cmp   $op1, $labl" %}
15950   ins_encode %{
15951     Label* L = $labl$$label;
15952     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15953     if (cond == Assembler::EQ || cond == Assembler::LS)
15954       __ cbzw($op1$$Register, *L);
15955     else
15956       __ cbnzw($op1$$Register, *L);
15957   %}
15958   ins_pipe(pipe_cmp_branch);
15959 %}
15960 
15961 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15962   match(If cmp (CmpUL op1 op2));
15963   effect(USE labl);
15964 
15965   ins_cost(BRANCH_COST);
15966   format %{ "cb$cmp   $op1, $labl" %}
15967   ins_encode %{
15968     Label* L = $labl$$label;
15969     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15970     if (cond == Assembler::EQ || cond == Assembler::LS)
15971       __ cbz($op1$$Register, *L);
15972     else
15973       __ cbnz($op1$$Register, *L);
15974   %}
15975   ins_pipe(pipe_cmp_branch);
15976 %}
15977 
15978 // Test bit and Branch
15979 
15980 // Patterns for short (< 32KiB) variants
15981 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15982   match(If cmp (CmpL op1 op2));
15983   effect(USE labl);
15984 
15985   ins_cost(BRANCH_COST);
15986   format %{ "cb$cmp   $op1, $labl # long" %}
15987   ins_encode %{
15988     Label* L = $labl$$label;
15989     Assembler::Condition cond =
15990       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15991     __ tbr(cond, $op1$$Register, 63, *L);
15992   %}
15993   ins_pipe(pipe_cmp_branch);
15994   ins_short_branch(1);
15995 %}
15996 
15997 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15998   match(If cmp (CmpI op1 op2));
15999   effect(USE labl);
16000 
16001   ins_cost(BRANCH_COST);
16002   format %{ "cb$cmp   $op1, $labl # int" %}
16003   ins_encode %{
16004     Label* L = $labl$$label;
16005     Assembler::Condition cond =
16006       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16007     __ tbr(cond, $op1$$Register, 31, *L);
16008   %}
16009   ins_pipe(pipe_cmp_branch);
16010   ins_short_branch(1);
16011 %}
16012 
16013 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16014   match(If cmp (CmpL (AndL op1 op2) op3));
16015   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16016   effect(USE labl);
16017 
16018   ins_cost(BRANCH_COST);
16019   format %{ "tb$cmp   $op1, $op2, $labl" %}
16020   ins_encode %{
16021     Label* L = $labl$$label;
16022     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16023     int bit = exact_log2_long($op2$$constant);
16024     __ tbr(cond, $op1$$Register, bit, *L);
16025   %}
16026   ins_pipe(pipe_cmp_branch);
16027   ins_short_branch(1);
16028 %}
16029 
16030 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16031   match(If cmp (CmpI (AndI op1 op2) op3));
16032   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16033   effect(USE labl);
16034 
16035   ins_cost(BRANCH_COST);
16036   format %{ "tb$cmp   $op1, $op2, $labl" %}
16037   ins_encode %{
16038     Label* L = $labl$$label;
16039     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16040     int bit = exact_log2((juint)$op2$$constant);
16041     __ tbr(cond, $op1$$Register, bit, *L);
16042   %}
16043   ins_pipe(pipe_cmp_branch);
16044   ins_short_branch(1);
16045 %}
16046 
16047 // And far variants
16048 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16049   match(If cmp (CmpL op1 op2));
16050   effect(USE labl);
16051 
16052   ins_cost(BRANCH_COST);
16053   format %{ "cb$cmp   $op1, $labl # long" %}
16054   ins_encode %{
16055     Label* L = $labl$$label;
16056     Assembler::Condition cond =
16057       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16058     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16059   %}
16060   ins_pipe(pipe_cmp_branch);
16061 %}
16062 
16063 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16064   match(If cmp (CmpI op1 op2));
16065   effect(USE labl);
16066 
16067   ins_cost(BRANCH_COST);
16068   format %{ "cb$cmp   $op1, $labl # int" %}
16069   ins_encode %{
16070     Label* L = $labl$$label;
16071     Assembler::Condition cond =
16072       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16073     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16074   %}
16075   ins_pipe(pipe_cmp_branch);
16076 %}
16077 
16078 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16079   match(If cmp (CmpL (AndL op1 op2) op3));
16080   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16081   effect(USE labl);
16082 
16083   ins_cost(BRANCH_COST);
16084   format %{ "tb$cmp   $op1, $op2, $labl" %}
16085   ins_encode %{
16086     Label* L = $labl$$label;
16087     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16088     int bit = exact_log2_long($op2$$constant);
16089     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16090   %}
16091   ins_pipe(pipe_cmp_branch);
16092 %}
16093 
16094 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16095   match(If cmp (CmpI (AndI op1 op2) op3));
16096   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16097   effect(USE labl);
16098 
16099   ins_cost(BRANCH_COST);
16100   format %{ "tb$cmp   $op1, $op2, $labl" %}
16101   ins_encode %{
16102     Label* L = $labl$$label;
16103     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16104     int bit = exact_log2((juint)$op2$$constant);
16105     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16106   %}
16107   ins_pipe(pipe_cmp_branch);
16108 %}
16109 
16110 // Test bits
16111 
16112 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16113   match(Set cr (CmpL (AndL op1 op2) op3));
16114   predicate(Assembler::operand_valid_for_logical_immediate
16115             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16116 
16117   ins_cost(INSN_COST);
16118   format %{ "tst $op1, $op2 # long" %}
16119   ins_encode %{
16120     __ tst($op1$$Register, $op2$$constant);
16121   %}
16122   ins_pipe(ialu_reg_reg);
16123 %}
16124 
16125 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16126   match(Set cr (CmpI (AndI op1 op2) op3));
16127   predicate(Assembler::operand_valid_for_logical_immediate
16128             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16129 
16130   ins_cost(INSN_COST);
16131   format %{ "tst $op1, $op2 # int" %}
16132   ins_encode %{
16133     __ tstw($op1$$Register, $op2$$constant);
16134   %}
16135   ins_pipe(ialu_reg_reg);
16136 %}
16137 
16138 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16139   match(Set cr (CmpL (AndL op1 op2) op3));
16140 
16141   ins_cost(INSN_COST);
16142   format %{ "tst $op1, $op2 # long" %}
16143   ins_encode %{
16144     __ tst($op1$$Register, $op2$$Register);
16145   %}
16146   ins_pipe(ialu_reg_reg);
16147 %}
16148 
16149 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16150   match(Set cr (CmpI (AndI op1 op2) op3));
16151 
16152   ins_cost(INSN_COST);
16153   format %{ "tstw $op1, $op2 # int" %}
16154   ins_encode %{
16155     __ tstw($op1$$Register, $op2$$Register);
16156   %}
16157   ins_pipe(ialu_reg_reg);
16158 %}
16159 
16160 
16161 // Conditional Far Branch
16162 // Conditional Far Branch Unsigned
16163 // TODO: fixme
16164 
16165 // counted loop end branch near
16166 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16167 %{
16168   match(CountedLoopEnd cmp cr);
16169 
16170   effect(USE lbl);
16171 
16172   ins_cost(BRANCH_COST);
16173   // short variant.
16174   // ins_short_branch(1);
16175   format %{ "b$cmp $lbl \t// counted loop end" %}
16176 
16177   ins_encode(aarch64_enc_br_con(cmp, lbl));
16178 
16179   ins_pipe(pipe_branch);
16180 %}
16181 
16182 // counted loop end branch near Unsigned
16183 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16184 %{
16185   match(CountedLoopEnd cmp cr);
16186 
16187   effect(USE lbl);
16188 
16189   ins_cost(BRANCH_COST);
16190   // short variant.
16191   // ins_short_branch(1);
16192   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16193 
16194   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16195 
16196   ins_pipe(pipe_branch);
16197 %}
16198 
16199 // counted loop end branch far
16200 // counted loop end branch far unsigned
16201 // TODO: fixme
16202 
16203 // ============================================================================
16204 // inlined locking and unlocking
16205 
16206 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16207 %{
16208   match(Set cr (FastLock object box));
16209   effect(TEMP tmp, TEMP tmp2);
16210 
16211   // TODO
16212   // identify correct cost
16213   ins_cost(5 * INSN_COST);
16214   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16215 
16216   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16217 
16218   ins_pipe(pipe_serial);
16219 %}
16220 
16221 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16222 %{
16223   match(Set cr (FastUnlock object box));
16224   effect(TEMP tmp, TEMP tmp2);
16225 
16226   ins_cost(5 * INSN_COST);
16227   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16228 
16229   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16230 
16231   ins_pipe(pipe_serial);
16232 %}
16233 
16234 
16235 // ============================================================================
16236 // Safepoint Instructions
16237 
16238 // TODO
16239 // provide a near and far version of this code
16240 
16241 instruct safePoint(rFlagsReg cr, iRegP poll)
16242 %{
16243   match(SafePoint poll);
16244   effect(KILL cr);
16245 
16246   format %{
16247     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16248   %}
16249   ins_encode %{
16250     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16251   %}
16252   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16253 %}
16254 
16255 
16256 // ============================================================================
16257 // Procedure Call/Return Instructions
16258 
16259 // Call Java Static Instruction
16260 
16261 instruct CallStaticJavaDirect(method meth)
16262 %{
16263   match(CallStaticJava);
16264 
16265   effect(USE meth);
16266 
16267   ins_cost(CALL_COST);
16268 
16269   format %{ "call,static $meth \t// ==> " %}
16270 
16271   ins_encode(aarch64_enc_java_static_call(meth),
16272              aarch64_enc_call_epilog);
16273 
16274   ins_pipe(pipe_class_call);
16275 %}
16276 
16277 // TO HERE
16278 
16279 // Call Java Dynamic Instruction
16280 instruct CallDynamicJavaDirect(method meth)
16281 %{
16282   match(CallDynamicJava);
16283 
16284   effect(USE meth);
16285 
16286   ins_cost(CALL_COST);
16287 
16288   format %{ "CALL,dynamic $meth \t// ==> " %}
16289 
16290   ins_encode(aarch64_enc_java_dynamic_call(meth),
16291              aarch64_enc_call_epilog);
16292 
16293   ins_pipe(pipe_class_call);
16294 %}
16295 
16296 // Call Runtime Instruction
16297 
16298 instruct CallRuntimeDirect(method meth)
16299 %{
16300   match(CallRuntime);
16301 
16302   effect(USE meth);
16303 
16304   ins_cost(CALL_COST);
16305 
16306   format %{ "CALL, runtime $meth" %}
16307 
16308   ins_encode( aarch64_enc_java_to_runtime(meth) );
16309 
16310   ins_pipe(pipe_class_call);
16311 %}
16312 
16313 // Call Runtime Instruction
16314 
16315 instruct CallLeafDirect(method meth)
16316 %{
16317   match(CallLeaf);
16318 
16319   effect(USE meth);
16320 
16321   ins_cost(CALL_COST);
16322 
16323   format %{ "CALL, runtime leaf $meth" %}
16324 
16325   ins_encode( aarch64_enc_java_to_runtime(meth) );
16326 
16327   ins_pipe(pipe_class_call);
16328 %}
16329 
16330 // Call Runtime Instruction
16331 
16332 instruct CallLeafNoFPDirect(method meth)
16333 %{
16334   match(CallLeafNoFP);
16335 
16336   effect(USE meth);
16337 
16338   ins_cost(CALL_COST);
16339 
16340   format %{ "CALL, runtime leaf nofp $meth" %}
16341 
16342   ins_encode( aarch64_enc_java_to_runtime(meth) );
16343 
16344   ins_pipe(pipe_class_call);
16345 %}
16346 
16347 instruct CallNativeDirect(method meth)
16348 %{
16349   match(CallNative);
16350 
16351   effect(USE meth);
16352 
16353   ins_cost(CALL_COST);
16354 
16355   format %{ "CALL, native $meth" %}
16356 
16357   ins_encode( aarch64_enc_java_to_runtime(meth) );
16358 
16359   ins_pipe(pipe_class_call);
16360 %}
16361 
16362 // Tail Call; Jump from runtime stub to Java code.
16363 // Also known as an 'interprocedural jump'.
16364 // Target of jump will eventually return to caller.
16365 // TailJump below removes the return address.
16366 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16367 %{
16368   match(TailCall jump_target method_ptr);
16369 
16370   ins_cost(CALL_COST);
16371 
16372   format %{ "br $jump_target\t# $method_ptr holds method" %}
16373 
16374   ins_encode(aarch64_enc_tail_call(jump_target));
16375 
16376   ins_pipe(pipe_class_call);
16377 %}
16378 
16379 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16380 %{
16381   match(TailJump jump_target ex_oop);
16382 
16383   ins_cost(CALL_COST);
16384 
16385   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16386 
16387   ins_encode(aarch64_enc_tail_jmp(jump_target));
16388 
16389   ins_pipe(pipe_class_call);
16390 %}
16391 
16392 // Create exception oop: created by stack-crawling runtime code.
16393 // Created exception is now available to this handler, and is setup
16394 // just prior to jumping to this handler. No code emitted.
16395 // TODO check
16396 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16397 instruct CreateException(iRegP_R0 ex_oop)
16398 %{
16399   match(Set ex_oop (CreateEx));
16400 
16401   format %{ " -- \t// exception oop; no code emitted" %}
16402 
16403   size(0);
16404 
16405   ins_encode( /*empty*/ );
16406 
16407   ins_pipe(pipe_class_empty);
16408 %}
16409 
16410 // Rethrow exception: The exception oop will come in the first
16411 // argument position. Then JUMP (not call) to the rethrow stub code.
16412 instruct RethrowException() %{
16413   match(Rethrow);
16414   ins_cost(CALL_COST);
16415 
16416   format %{ "b rethrow_stub" %}
16417 
16418   ins_encode( aarch64_enc_rethrow() );
16419 
16420   ins_pipe(pipe_class_call);
16421 %}
16422 
16423 
16424 // Return Instruction
16425 // epilog node loads ret address into lr as part of frame pop
16426 instruct Ret()
16427 %{
16428   match(Return);
16429 
16430   format %{ "ret\t// return register" %}
16431 
16432   ins_encode( aarch64_enc_ret() );
16433 
16434   ins_pipe(pipe_branch);
16435 %}
16436 
16437 // Die now.
16438 instruct ShouldNotReachHere() %{
16439   match(Halt);
16440 
16441   ins_cost(CALL_COST);
16442   format %{ "ShouldNotReachHere" %}
16443 
16444   ins_encode %{
16445     if (is_reachable()) {
16446       __ stop(_halt_reason);
16447     }
16448   %}
16449 
16450   ins_pipe(pipe_class_default);
16451 %}
16452 
16453 // ============================================================================
16454 // Partial Subtype Check
16455 //
16456 // superklass array for an instance of the superklass.  Set a hidden
16457 // internal cache on a hit (cache is checked with exposed code in
16458 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16459 // encoding ALSO sets flags.
16460 
16461 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16462 %{
16463   match(Set result (PartialSubtypeCheck sub super));
16464   effect(KILL cr, KILL temp);
16465 
16466   ins_cost(1100);  // slightly larger than the next version
16467   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16468 
16469   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16470 
16471   opcode(0x1); // Force zero of result reg on hit
16472 
16473   ins_pipe(pipe_class_memory);
16474 %}
16475 
16476 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16477 %{
16478   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16479   effect(KILL temp, KILL result);
16480 
16481   ins_cost(1100);  // slightly larger than the next version
16482   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16483 
16484   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16485 
16486   opcode(0x0); // Don't zero result reg on hit
16487 
16488   ins_pipe(pipe_class_memory);
16489 %}
16490 
16491 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16492                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16493 %{
16494   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16495   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16496   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16497 
16498   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16499   ins_encode %{
16500     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16501     __ string_compare($str1$$Register, $str2$$Register,
16502                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16503                       $tmp1$$Register, $tmp2$$Register,
16504                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16505   %}
16506   ins_pipe(pipe_class_memory);
16507 %}
16508 
16509 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16510                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16511 %{
16512   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16513   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16514   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16515 
16516   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16517   ins_encode %{
16518     __ string_compare($str1$$Register, $str2$$Register,
16519                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16520                       $tmp1$$Register, $tmp2$$Register,
16521                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16522   %}
16523   ins_pipe(pipe_class_memory);
16524 %}
16525 
16526 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16527                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16528                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16529 %{
16530   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16531   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16532   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16533          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16534 
16535   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16536   ins_encode %{
16537     __ string_compare($str1$$Register, $str2$$Register,
16538                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16539                       $tmp1$$Register, $tmp2$$Register,
16540                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16541                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16542   %}
16543   ins_pipe(pipe_class_memory);
16544 %}
16545 
16546 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16547                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16548                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16549 %{
16550   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16551   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16552   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16553          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16554 
16555   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16556   ins_encode %{
16557     __ string_compare($str1$$Register, $str2$$Register,
16558                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16559                       $tmp1$$Register, $tmp2$$Register,
16560                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16561                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16562   %}
16563   ins_pipe(pipe_class_memory);
16564 %}
16565 
16566 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16567        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16568        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16569 %{
16570   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16571   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16572   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16573          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16574   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16575 
16576   ins_encode %{
16577     __ string_indexof($str1$$Register, $str2$$Register,
16578                       $cnt1$$Register, $cnt2$$Register,
16579                       $tmp1$$Register, $tmp2$$Register,
16580                       $tmp3$$Register, $tmp4$$Register,
16581                       $tmp5$$Register, $tmp6$$Register,
16582                       -1, $result$$Register, StrIntrinsicNode::UU);
16583   %}
16584   ins_pipe(pipe_class_memory);
16585 %}
16586 
16587 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16588        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16589        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16590 %{
16591   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16592   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16593   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16594          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16595   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16596 
16597   ins_encode %{
16598     __ string_indexof($str1$$Register, $str2$$Register,
16599                       $cnt1$$Register, $cnt2$$Register,
16600                       $tmp1$$Register, $tmp2$$Register,
16601                       $tmp3$$Register, $tmp4$$Register,
16602                       $tmp5$$Register, $tmp6$$Register,
16603                       -1, $result$$Register, StrIntrinsicNode::LL);
16604   %}
16605   ins_pipe(pipe_class_memory);
16606 %}
16607 
16608 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16609        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16610        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16611 %{
16612   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16613   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16614   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16615          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16616   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16617 
16618   ins_encode %{
16619     __ string_indexof($str1$$Register, $str2$$Register,
16620                       $cnt1$$Register, $cnt2$$Register,
16621                       $tmp1$$Register, $tmp2$$Register,
16622                       $tmp3$$Register, $tmp4$$Register,
16623                       $tmp5$$Register, $tmp6$$Register,
16624                       -1, $result$$Register, StrIntrinsicNode::UL);
16625   %}
16626   ins_pipe(pipe_class_memory);
16627 %}
16628 
16629 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16630                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16631                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16632 %{
16633   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16634   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16635   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16636          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16637   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16638 
16639   ins_encode %{
16640     int icnt2 = (int)$int_cnt2$$constant;
16641     __ string_indexof($str1$$Register, $str2$$Register,
16642                       $cnt1$$Register, zr,
16643                       $tmp1$$Register, $tmp2$$Register,
16644                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16645                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16646   %}
16647   ins_pipe(pipe_class_memory);
16648 %}
16649 
16650 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16651                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16652                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16653 %{
16654   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16655   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16656   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16657          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16658   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16659 
16660   ins_encode %{
16661     int icnt2 = (int)$int_cnt2$$constant;
16662     __ string_indexof($str1$$Register, $str2$$Register,
16663                       $cnt1$$Register, zr,
16664                       $tmp1$$Register, $tmp2$$Register,
16665                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16666                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16667   %}
16668   ins_pipe(pipe_class_memory);
16669 %}
16670 
16671 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16672                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16673                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16674 %{
16675   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16676   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16677   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16678          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16679   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16680 
16681   ins_encode %{
16682     int icnt2 = (int)$int_cnt2$$constant;
16683     __ string_indexof($str1$$Register, $str2$$Register,
16684                       $cnt1$$Register, zr,
16685                       $tmp1$$Register, $tmp2$$Register,
16686                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16687                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16688   %}
16689   ins_pipe(pipe_class_memory);
16690 %}
16691 
16692 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16693                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16694                              iRegINoSp tmp3, rFlagsReg cr)
16695 %{
16696   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16697   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16698   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16699          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16700 
16701   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16702 
16703   ins_encode %{
16704     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16705                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16706                            $tmp3$$Register);
16707   %}
16708   ins_pipe(pipe_class_memory);
16709 %}
16710 
16711 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16712                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16713                               iRegINoSp tmp3, rFlagsReg cr)
16714 %{
16715   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16716   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16717   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16718          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16719 
16720   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16721 
16722   ins_encode %{
16723     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16724                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16725                             $tmp3$$Register);
16726   %}
16727   ins_pipe(pipe_class_memory);
16728 %}
16729 
16730 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16731                         iRegI_R0 result, rFlagsReg cr)
16732 %{
16733   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16734   match(Set result (StrEquals (Binary str1 str2) cnt));
16735   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16736 
16737   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16738   ins_encode %{
16739     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16740     __ string_equals($str1$$Register, $str2$$Register,
16741                      $result$$Register, $cnt$$Register, 1);
16742   %}
16743   ins_pipe(pipe_class_memory);
16744 %}
16745 
16746 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16747                         iRegI_R0 result, rFlagsReg cr)
16748 %{
16749   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16750   match(Set result (StrEquals (Binary str1 str2) cnt));
16751   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16752 
16753   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16754   ins_encode %{
16755     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16756     __ string_equals($str1$$Register, $str2$$Register,
16757                      $result$$Register, $cnt$$Register, 2);
16758   %}
16759   ins_pipe(pipe_class_memory);
16760 %}
16761 
16762 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16763                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16764                        iRegP_R10 tmp, rFlagsReg cr)
16765 %{
16766   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16767   match(Set result (AryEq ary1 ary2));
16768   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16769 
16770   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16771   ins_encode %{
16772     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16773                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16774                                    $result$$Register, $tmp$$Register, 1);
16775     if (tpc == NULL) {
16776       ciEnv::current()->record_failure("CodeCache is full");
16777       return;
16778     }
16779   %}
16780   ins_pipe(pipe_class_memory);
16781 %}
16782 
16783 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16784                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16785                        iRegP_R10 tmp, rFlagsReg cr)
16786 %{
16787   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16788   match(Set result (AryEq ary1 ary2));
16789   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16790 
16791   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16792   ins_encode %{
16793     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16794                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16795                                    $result$$Register, $tmp$$Register, 2);
16796     if (tpc == NULL) {
16797       ciEnv::current()->record_failure("CodeCache is full");
16798       return;
16799     }
16800   %}
16801   ins_pipe(pipe_class_memory);
16802 %}
16803 
16804 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16805 %{
16806   match(Set result (HasNegatives ary1 len));
16807   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16808   format %{ "has negatives byte[] $ary1,$len -> $result" %}
16809   ins_encode %{
16810     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
16811     if (tpc == NULL) {
16812       ciEnv::current()->record_failure("CodeCache is full");
16813       return;
16814     }
16815   %}
16816   ins_pipe( pipe_slow );
16817 %}
16818 
16819 // fast char[] to byte[] compression
16820 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16821                          vRegD_V0 tmp1, vRegD_V1 tmp2,
16822                          vRegD_V2 tmp3, vRegD_V3 tmp4,
16823                          iRegI_R0 result, rFlagsReg cr)
16824 %{
16825   match(Set result (StrCompressedCopy src (Binary dst len)));
16826   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16827 
16828   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
16829   ins_encode %{
16830     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16831                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16832                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
16833                            $result$$Register);
16834   %}
16835   ins_pipe( pipe_slow );
16836 %}
16837 
16838 // fast byte[] to char[] inflation
16839 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16840                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16841 %{
16842   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16843   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16844 
16845   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16846   ins_encode %{
16847     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16848                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16849                                         $tmp3$$FloatRegister, $tmp4$$Register);
16850     if (tpc == NULL) {
16851       ciEnv::current()->record_failure("CodeCache is full");
16852       return;
16853     }
16854   %}
16855   ins_pipe(pipe_class_memory);
16856 %}
16857 
16858 // encode char[] to byte[] in ISO_8859_1
16859 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16860                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
16861                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
16862                           iRegI_R0 result, rFlagsReg cr)
16863 %{
16864   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16865   match(Set result (EncodeISOArray src (Binary dst len)));
16866   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
16867          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
16868 
16869   format %{ "Encode array $src,$dst,$len -> $result" %}
16870   ins_encode %{
16871     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16872          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
16873          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
16874   %}
16875   ins_pipe( pipe_class_memory );
16876 %}
16877 
16878 // ============================================================================
16879 // This name is KNOWN by the ADLC and cannot be changed.
16880 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16881 // for this guy.
16882 instruct tlsLoadP(thread_RegP dst)
16883 %{
16884   match(Set dst (ThreadLocal));
16885 
16886   ins_cost(0);
16887 
16888   format %{ " -- \t// $dst=Thread::current(), empty" %}
16889 
16890   size(0);
16891 
16892   ins_encode( /*empty*/ );
16893 
16894   ins_pipe(pipe_class_empty);
16895 %}
16896 
16897 //----------PEEPHOLE RULES-----------------------------------------------------
16898 // These must follow all instruction definitions as they use the names
16899 // defined in the instructions definitions.
16900 //
16901 // peepmatch ( root_instr_name [preceding_instruction]* );
16902 //
16903 // peepconstraint %{
16904 // (instruction_number.operand_name relational_op instruction_number.operand_name
16905 //  [, ...] );
16906 // // instruction numbers are zero-based using left to right order in peepmatch
16907 //
16908 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
16909 // // provide an instruction_number.operand_name for each operand that appears
16910 // // in the replacement instruction's match rule
16911 //
16912 // ---------VM FLAGS---------------------------------------------------------
16913 //
16914 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16915 //
16916 // Each peephole rule is given an identifying number starting with zero and
16917 // increasing by one in the order seen by the parser.  An individual peephole
16918 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16919 // on the command-line.
16920 //
16921 // ---------CURRENT LIMITATIONS----------------------------------------------
16922 //
16923 // Only match adjacent instructions in same basic block
16924 // Only equality constraints
16925 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16926 // Only one replacement instruction
16927 //
16928 // ---------EXAMPLE----------------------------------------------------------
16929 //
16930 // // pertinent parts of existing instructions in architecture description
16931 // instruct movI(iRegINoSp dst, iRegI src)
16932 // %{
16933 //   match(Set dst (CopyI src));
16934 // %}
16935 //
16936 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16937 // %{
16938 //   match(Set dst (AddI dst src));
16939 //   effect(KILL cr);
16940 // %}
16941 //
16942 // // Change (inc mov) to lea
16943 // peephole %{
16944 //   // increment preceeded by register-register move
16945 //   peepmatch ( incI_iReg movI );
16946 //   // require that the destination register of the increment
16947 //   // match the destination register of the move
16948 //   peepconstraint ( 0.dst == 1.dst );
16949 //   // construct a replacement instruction that sets
16950 //   // the destination to ( move's source register + one )
16951 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16952 // %}
16953 //
16954 
16955 // Implementation no longer uses movX instructions since
16956 // machine-independent system no longer uses CopyX nodes.
16957 //
16958 // peephole
16959 // %{
16960 //   peepmatch (incI_iReg movI);
16961 //   peepconstraint (0.dst == 1.dst);
16962 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16963 // %}
16964 
16965 // peephole
16966 // %{
16967 //   peepmatch (decI_iReg movI);
16968 //   peepconstraint (0.dst == 1.dst);
16969 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16970 // %}
16971 
16972 // peephole
16973 // %{
16974 //   peepmatch (addI_iReg_imm movI);
16975 //   peepconstraint (0.dst == 1.dst);
16976 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16977 // %}
16978 
16979 // peephole
16980 // %{
16981 //   peepmatch (incL_iReg movL);
16982 //   peepconstraint (0.dst == 1.dst);
16983 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16984 // %}
16985 
16986 // peephole
16987 // %{
16988 //   peepmatch (decL_iReg movL);
16989 //   peepconstraint (0.dst == 1.dst);
16990 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16991 // %}
16992 
16993 // peephole
16994 // %{
16995 //   peepmatch (addL_iReg_imm movL);
16996 //   peepconstraint (0.dst == 1.dst);
16997 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16998 // %}
16999 
17000 // peephole
17001 // %{
17002 //   peepmatch (addP_iReg_imm movP);
17003 //   peepconstraint (0.dst == 1.dst);
17004 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17005 // %}
17006 
17007 // // Change load of spilled value to only a spill
17008 // instruct storeI(memory mem, iRegI src)
17009 // %{
17010 //   match(Set mem (StoreI mem src));
17011 // %}
17012 //
17013 // instruct loadI(iRegINoSp dst, memory mem)
17014 // %{
17015 //   match(Set dst (LoadI mem));
17016 // %}
17017 //
17018 
17019 //----------SMARTSPILL RULES---------------------------------------------------
17020 // These must follow all instruction definitions as they use the names
17021 // defined in the instructions definitions.
17022 
17023 // Local Variables:
17024 // mode: c++
17025 // End: