1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return 4 * NativeInstruction::instruction_size;
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298   bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // predicate controlling addressing modes
 1315   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1316 %}
 1317 
 1318 source %{
 1319 
 1320   // Derived RegMask with conditionally allocatable registers
 1321 
 1322   void PhaseOutput::pd_perform_mach_node_analysis() {
 1323   }
 1324 
 1325   int MachNode::pd_alignment_required() const {
 1326     return 1;
 1327   }
 1328 
 1329   int MachNode::compute_padding(int current_offset) const {
 1330     return 0;
 1331   }
 1332 
 1333   RegMask _ANY_REG32_mask;
 1334   RegMask _ANY_REG_mask;
 1335   RegMask _PTR_REG_mask;
 1336   RegMask _NO_SPECIAL_REG32_mask;
 1337   RegMask _NO_SPECIAL_REG_mask;
 1338   RegMask _NO_SPECIAL_PTR_REG_mask;
 1339 
 1340   void reg_mask_init() {
 1341     // We derive below RegMask(s) from the ones which are auto-generated from
 1342     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1343     // registers conditionally reserved.
 1344 
 1345     _ANY_REG32_mask = _ALL_REG32_mask;
 1346     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1347 
 1348     _ANY_REG_mask = _ALL_REG_mask;
 1349 
 1350     _PTR_REG_mask = _ALL_REG_mask;
 1351 
 1352     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1353     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1354 
 1355     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1356     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1357 
 1358     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     // r27 is not allocatable when compressed oops is on and heapbase is not
 1362     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1363     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1364       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1365       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1366       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1367     }
 1368 
 1369     // r29 is not allocatable when PreserveFramePointer is on
 1370     if (PreserveFramePointer) {
 1371       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1372       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1373       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1374     }
 1375   }
 1376 
 1377   // Optimizaton of volatile gets and puts
 1378   // -------------------------------------
 1379   //
 1380   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1381   // use to implement volatile reads and writes. For a volatile read
 1382   // we simply need
 1383   //
 1384   //   ldar<x>
 1385   //
 1386   // and for a volatile write we need
 1387   //
 1388   //   stlr<x>
 1389   //
 1390   // Alternatively, we can implement them by pairing a normal
 1391   // load/store with a memory barrier. For a volatile read we need
 1392   //
 1393   //   ldr<x>
 1394   //   dmb ishld
 1395   //
 1396   // for a volatile write
 1397   //
 1398   //   dmb ish
 1399   //   str<x>
 1400   //   dmb ish
 1401   //
 1402   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1403   // sequences. These are normally translated to an instruction
 1404   // sequence like the following
 1405   //
 1406   //   dmb      ish
 1407   // retry:
 1408   //   ldxr<x>   rval raddr
 1409   //   cmp       rval rold
 1410   //   b.ne done
 1411   //   stlxr<x>  rval, rnew, rold
 1412   //   cbnz      rval retry
 1413   // done:
 1414   //   cset      r0, eq
 1415   //   dmb ishld
 1416   //
 1417   // Note that the exclusive store is already using an stlxr
 1418   // instruction. That is required to ensure visibility to other
 1419   // threads of the exclusive write (assuming it succeeds) before that
 1420   // of any subsequent writes.
 1421   //
 1422   // The following instruction sequence is an improvement on the above
 1423   //
 1424   // retry:
 1425   //   ldaxr<x>  rval raddr
 1426   //   cmp       rval rold
 1427   //   b.ne done
 1428   //   stlxr<x>  rval, rnew, rold
 1429   //   cbnz      rval retry
 1430   // done:
 1431   //   cset      r0, eq
 1432   //
 1433   // We don't need the leading dmb ish since the stlxr guarantees
 1434   // visibility of prior writes in the case that the swap is
 1435   // successful. Crucially we don't have to worry about the case where
 1436   // the swap is not successful since no valid program should be
 1437   // relying on visibility of prior changes by the attempting thread
 1438   // in the case where the CAS fails.
 1439   //
 1440   // Similarly, we don't need the trailing dmb ishld if we substitute
 1441   // an ldaxr instruction since that will provide all the guarantees we
 1442   // require regarding observation of changes made by other threads
 1443   // before any change to the CAS address observed by the load.
 1444   //
 1445   // In order to generate the desired instruction sequence we need to
 1446   // be able to identify specific 'signature' ideal graph node
 1447   // sequences which i) occur as a translation of a volatile reads or
 1448   // writes or CAS operations and ii) do not occur through any other
 1449   // translation or graph transformation. We can then provide
 1450   // alternative aldc matching rules which translate these node
 1451   // sequences to the desired machine code sequences. Selection of the
 1452   // alternative rules can be implemented by predicates which identify
 1453   // the relevant node sequences.
 1454   //
 1455   // The ideal graph generator translates a volatile read to the node
 1456   // sequence
 1457   //
 1458   //   LoadX[mo_acquire]
 1459   //   MemBarAcquire
 1460   //
 1461   // As a special case when using the compressed oops optimization we
 1462   // may also see this variant
 1463   //
 1464   //   LoadN[mo_acquire]
 1465   //   DecodeN
 1466   //   MemBarAcquire
 1467   //
 1468   // A volatile write is translated to the node sequence
 1469   //
 1470   //   MemBarRelease
 1471   //   StoreX[mo_release] {CardMark}-optional
 1472   //   MemBarVolatile
 1473   //
 1474   // n.b. the above node patterns are generated with a strict
 1475   // 'signature' configuration of input and output dependencies (see
 1476   // the predicates below for exact details). The card mark may be as
 1477   // simple as a few extra nodes or, in a few GC configurations, may
 1478   // include more complex control flow between the leading and
 1479   // trailing memory barriers. However, whatever the card mark
 1480   // configuration these signatures are unique to translated volatile
 1481   // reads/stores -- they will not appear as a result of any other
 1482   // bytecode translation or inlining nor as a consequence of
 1483   // optimizing transforms.
 1484   //
 1485   // We also want to catch inlined unsafe volatile gets and puts and
 1486   // be able to implement them using either ldar<x>/stlr<x> or some
 1487   // combination of ldr<x>/stlr<x> and dmb instructions.
 1488   //
 1489   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1490   // normal volatile put node sequence containing an extra cpuorder
 1491   // membar
 1492   //
 1493   //   MemBarRelease
 1494   //   MemBarCPUOrder
 1495   //   StoreX[mo_release] {CardMark}-optional
 1496   //   MemBarCPUOrder
 1497   //   MemBarVolatile
 1498   //
 1499   // n.b. as an aside, a cpuorder membar is not itself subject to
 1500   // matching and translation by adlc rules.  However, the rule
 1501   // predicates need to detect its presence in order to correctly
 1502   // select the desired adlc rules.
 1503   //
 1504   // Inlined unsafe volatile gets manifest as a slightly different
 1505   // node sequence to a normal volatile get because of the
 1506   // introduction of some CPUOrder memory barriers to bracket the
 1507   // Load. However, but the same basic skeleton of a LoadX feeding a
 1508   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1509   // present
 1510   //
 1511   //   MemBarCPUOrder
 1512   //        ||       \\
 1513   //   MemBarCPUOrder LoadX[mo_acquire]
 1514   //        ||            |
 1515   //        ||       {DecodeN} optional
 1516   //        ||       /
 1517   //     MemBarAcquire
 1518   //
 1519   // In this case the acquire membar does not directly depend on the
 1520   // load. However, we can be sure that the load is generated from an
 1521   // inlined unsafe volatile get if we see it dependent on this unique
 1522   // sequence of membar nodes. Similarly, given an acquire membar we
 1523   // can know that it was added because of an inlined unsafe volatile
 1524   // get if it is fed and feeds a cpuorder membar and if its feed
 1525   // membar also feeds an acquiring load.
 1526   //
 1527   // Finally an inlined (Unsafe) CAS operation is translated to the
 1528   // following ideal graph
 1529   //
 1530   //   MemBarRelease
 1531   //   MemBarCPUOrder
 1532   //   CompareAndSwapX {CardMark}-optional
 1533   //   MemBarCPUOrder
 1534   //   MemBarAcquire
 1535   //
 1536   // So, where we can identify these volatile read and write
 1537   // signatures we can choose to plant either of the above two code
 1538   // sequences. For a volatile read we can simply plant a normal
 1539   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1540   // also choose to inhibit translation of the MemBarAcquire and
 1541   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1542   //
 1543   // When we recognise a volatile store signature we can choose to
 1544   // plant at a dmb ish as a translation for the MemBarRelease, a
 1545   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1546   // Alternatively, we can inhibit translation of the MemBarRelease
 1547   // and MemBarVolatile and instead plant a simple stlr<x>
 1548   // instruction.
 1549   //
 1550   // when we recognise a CAS signature we can choose to plant a dmb
 1551   // ish as a translation for the MemBarRelease, the conventional
 1552   // macro-instruction sequence for the CompareAndSwap node (which
 1553   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1554   // Alternatively, we can elide generation of the dmb instructions
 1555   // and plant the alternative CompareAndSwap macro-instruction
 1556   // sequence (which uses ldaxr<x>).
 1557   //
 1558   // Of course, the above only applies when we see these signature
 1559   // configurations. We still want to plant dmb instructions in any
 1560   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1561   // MemBarVolatile. For example, at the end of a constructor which
 1562   // writes final/volatile fields we will see a MemBarRelease
 1563   // instruction and this needs a 'dmb ish' lest we risk the
 1564   // constructed object being visible without making the
 1565   // final/volatile field writes visible.
 1566   //
 1567   // n.b. the translation rules below which rely on detection of the
 1568   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1569   // If we see anything other than the signature configurations we
 1570   // always just translate the loads and stores to ldr<x> and str<x>
 1571   // and translate acquire, release and volatile membars to the
 1572   // relevant dmb instructions.
 1573   //
 1574 
 1575   // is_CAS(int opcode, bool maybe_volatile)
 1576   //
 1577   // return true if opcode is one of the possible CompareAndSwapX
 1578   // values otherwise false.
 1579 
 1580   bool is_CAS(int opcode, bool maybe_volatile)
 1581   {
 1582     switch(opcode) {
 1583       // We handle these
 1584     case Op_CompareAndSwapI:
 1585     case Op_CompareAndSwapL:
 1586     case Op_CompareAndSwapP:
 1587     case Op_CompareAndSwapN:
 1588     case Op_ShenandoahCompareAndSwapP:
 1589     case Op_ShenandoahCompareAndSwapN:
 1590     case Op_CompareAndSwapB:
 1591     case Op_CompareAndSwapS:
 1592     case Op_GetAndSetI:
 1593     case Op_GetAndSetL:
 1594     case Op_GetAndSetP:
 1595     case Op_GetAndSetN:
 1596     case Op_GetAndAddI:
 1597     case Op_GetAndAddL:
 1598       return true;
 1599     case Op_CompareAndExchangeI:
 1600     case Op_CompareAndExchangeN:
 1601     case Op_CompareAndExchangeB:
 1602     case Op_CompareAndExchangeS:
 1603     case Op_CompareAndExchangeL:
 1604     case Op_CompareAndExchangeP:
 1605     case Op_WeakCompareAndSwapB:
 1606     case Op_WeakCompareAndSwapS:
 1607     case Op_WeakCompareAndSwapI:
 1608     case Op_WeakCompareAndSwapL:
 1609     case Op_WeakCompareAndSwapP:
 1610     case Op_WeakCompareAndSwapN:
 1611     case Op_ShenandoahWeakCompareAndSwapP:
 1612     case Op_ShenandoahWeakCompareAndSwapN:
 1613     case Op_ShenandoahCompareAndExchangeP:
 1614     case Op_ShenandoahCompareAndExchangeN:
 1615       return maybe_volatile;
 1616     default:
 1617       return false;
 1618     }
 1619   }
 1620 
 1621   // helper to determine the maximum number of Phi nodes we may need to
 1622   // traverse when searching from a card mark membar for the merge mem
 1623   // feeding a trailing membar or vice versa
 1624 
 1625 // predicates controlling emit of ldr<x>/ldar<x>
 1626 
 1627 bool unnecessary_acquire(const Node *barrier)
 1628 {
 1629   assert(barrier->is_MemBar(), "expecting a membar");
 1630 
 1631   MemBarNode* mb = barrier->as_MemBar();
 1632 
 1633   if (mb->trailing_load()) {
 1634     return true;
 1635   }
 1636 
 1637   if (mb->trailing_load_store()) {
 1638     Node* load_store = mb->in(MemBarNode::Precedent);
 1639     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1640     return is_CAS(load_store->Opcode(), true);
 1641   }
 1642 
 1643   return false;
 1644 }
 1645 
 1646 bool needs_acquiring_load(const Node *n)
 1647 {
 1648   assert(n->is_Load(), "expecting a load");
 1649   LoadNode *ld = n->as_Load();
 1650   return ld->is_acquire();
 1651 }
 1652 
 1653 bool unnecessary_release(const Node *n)
 1654 {
 1655   assert((n->is_MemBar() &&
 1656           n->Opcode() == Op_MemBarRelease),
 1657          "expecting a release membar");
 1658 
 1659   MemBarNode *barrier = n->as_MemBar();
 1660   if (!barrier->leading()) {
 1661     return false;
 1662   } else {
 1663     Node* trailing = barrier->trailing_membar();
 1664     MemBarNode* trailing_mb = trailing->as_MemBar();
 1665     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1666     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1667 
 1668     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1669     if (mem->is_Store()) {
 1670       assert(mem->as_Store()->is_release(), "");
 1671       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1672       return true;
 1673     } else {
 1674       assert(mem->is_LoadStore(), "");
 1675       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1676       return is_CAS(mem->Opcode(), true);
 1677     }
 1678   }
 1679   return false;
 1680 }
 1681 
 1682 bool unnecessary_volatile(const Node *n)
 1683 {
 1684   // assert n->is_MemBar();
 1685   MemBarNode *mbvol = n->as_MemBar();
 1686 
 1687   bool release = mbvol->trailing_store();
 1688   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1689 #ifdef ASSERT
 1690   if (release) {
 1691     Node* leading = mbvol->leading_membar();
 1692     assert(leading->Opcode() == Op_MemBarRelease, "");
 1693     assert(leading->as_MemBar()->leading_store(), "");
 1694     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1695   }
 1696 #endif
 1697 
 1698   return release;
 1699 }
 1700 
 1701 // predicates controlling emit of str<x>/stlr<x>
 1702 
 1703 bool needs_releasing_store(const Node *n)
 1704 {
 1705   // assert n->is_Store();
 1706   StoreNode *st = n->as_Store();
 1707   return st->trailing_membar() != NULL;
 1708 }
 1709 
 1710 // predicate controlling translation of CAS
 1711 //
 1712 // returns true if CAS needs to use an acquiring load otherwise false
 1713 
 1714 bool needs_acquiring_load_exclusive(const Node *n)
 1715 {
 1716   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1717   LoadStoreNode* ldst = n->as_LoadStore();
 1718   if (is_CAS(n->Opcode(), false)) {
 1719     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1720   } else {
 1721     return ldst->trailing_membar() != NULL;
 1722   }
 1723 
 1724   // so we can just return true here
 1725   return true;
 1726 }
 1727 
 1728 #define __ _masm.
 1729 
 1730 // advance declarations for helper functions to convert register
 1731 // indices to register objects
 1732 
 1733 // the ad file has to provide implementations of certain methods
 1734 // expected by the generic code
 1735 //
 1736 // REQUIRED FUNCTIONALITY
 1737 
 1738 //=============================================================================
 1739 
 1740 // !!!!! Special hack to get all types of calls to specify the byte offset
 1741 //       from the start of the call to the point where the return address
 1742 //       will point.
 1743 
 1744 int MachCallStaticJavaNode::ret_addr_offset()
 1745 {
 1746   // call should be a simple bl
 1747   int off = 4;
 1748   return off;
 1749 }
 1750 
 1751 int MachCallDynamicJavaNode::ret_addr_offset()
 1752 {
 1753   return 16; // movz, movk, movk, bl
 1754 }
 1755 
 1756 int MachCallRuntimeNode::ret_addr_offset() {
 1757   // for generated stubs the call will be
 1758   //   bl(addr)
 1759   // or with far branches
 1760   //   bl(trampoline_stub)
 1761   // for real runtime callouts it will be six instructions
 1762   // see aarch64_enc_java_to_runtime
 1763   //   adr(rscratch2, retaddr)
 1764   //   lea(rscratch1, RuntimeAddress(addr)
 1765   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1766   //   blr(rscratch1)
 1767   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1768   if (cb) {
 1769     return 1 * NativeInstruction::instruction_size;
 1770   } else {
 1771     return 6 * NativeInstruction::instruction_size;
 1772   }
 1773 }
 1774 
 1775 int MachCallNativeNode::ret_addr_offset() {
 1776   // This is implemented using aarch64_enc_java_to_runtime as above.
 1777   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1778   if (cb) {
 1779     return 1 * NativeInstruction::instruction_size;
 1780   } else {
 1781     return 6 * NativeInstruction::instruction_size;
 1782   }
 1783 }
 1784 
 1785 //=============================================================================
 1786 
 1787 #ifndef PRODUCT
 1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1789   st->print("BREAKPOINT");
 1790 }
 1791 #endif
 1792 
 1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1794   C2_MacroAssembler _masm(&cbuf);
 1795   __ brk(0);
 1796 }
 1797 
 1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1799   return MachNode::size(ra_);
 1800 }
 1801 
 1802 //=============================================================================
 1803 
 1804 #ifndef PRODUCT
 1805   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1806     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1807   }
 1808 #endif
 1809 
 1810   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1811     C2_MacroAssembler _masm(&cbuf);
 1812     for (int i = 0; i < _count; i++) {
 1813       __ nop();
 1814     }
 1815   }
 1816 
 1817   uint MachNopNode::size(PhaseRegAlloc*) const {
 1818     return _count * NativeInstruction::instruction_size;
 1819   }
 1820 
 1821 //=============================================================================
 1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1823 
 1824 int ConstantTable::calculate_table_base_offset() const {
 1825   return 0;  // absolute addressing, no offset
 1826 }
 1827 
 1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1830   ShouldNotReachHere();
 1831 }
 1832 
 1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1834   // Empty encoding
 1835 }
 1836 
 1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1838   return 0;
 1839 }
 1840 
 1841 #ifndef PRODUCT
 1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1843   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1844 }
 1845 #endif
 1846 
 1847 #ifndef PRODUCT
 1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1849   Compile* C = ra_->C;
 1850 
 1851   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1852 
 1853   if (C->output()->need_stack_bang(framesize))
 1854     st->print("# stack bang size=%d\n\t", framesize);
 1855 
 1856   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1857     st->print("sub  sp, sp, #%d\n\t", framesize);
 1858     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1859     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1860   } else {
 1861     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1862     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1863     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1864     st->print("sub  sp, sp, rscratch1");
 1865   }
 1866   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1867     st->print("\n\t");
 1868     st->print("ldr  rscratch1, [guard]\n\t");
 1869     st->print("dmb ishld\n\t");
 1870     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1871     st->print("cmp  rscratch1, rscratch2\n\t");
 1872     st->print("b.eq skip");
 1873     st->print("\n\t");
 1874     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1875     st->print("b skip\n\t");
 1876     st->print("guard: int\n\t");
 1877     st->print("\n\t");
 1878     st->print("skip:\n\t");
 1879   }
 1880 }
 1881 #endif
 1882 
 1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1884   Compile* C = ra_->C;
 1885   C2_MacroAssembler _masm(&cbuf);
 1886 
 1887   // n.b. frame size includes space for return pc and rfp
 1888   const int framesize = C->output()->frame_size_in_bytes();
 1889 
 1890   // insert a nop at the start of the prolog so we can patch in a
 1891   // branch if we need to invalidate the method later
 1892   __ nop();
 1893 
 1894   if (C->clinit_barrier_on_entry()) {
 1895     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1896 
 1897     Label L_skip_barrier;
 1898 
 1899     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1900     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1901     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1902     __ bind(L_skip_barrier);
 1903   }
 1904 
 1905   if (C->max_vector_size() > 0) {
 1906     __ reinitialize_ptrue();
 1907   }
 1908 
 1909   int bangsize = C->output()->bang_size_in_bytes();
 1910   if (C->output()->need_stack_bang(bangsize))
 1911     __ generate_stack_overflow_check(bangsize);
 1912 
 1913   __ build_frame(framesize);
 1914 
 1915   if (C->stub_function() == NULL) {
 1916     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1917     bs->nmethod_entry_barrier(&_masm);
 1918   }
 1919 
 1920   if (VerifyStackAtCalls) {
 1921     Unimplemented();
 1922   }
 1923 
 1924   C->output()->set_frame_complete(cbuf.insts_size());
 1925 
 1926   if (C->has_mach_constant_base_node()) {
 1927     // NOTE: We set the table base offset here because users might be
 1928     // emitted before MachConstantBaseNode.
 1929     ConstantTable& constant_table = C->output()->constant_table();
 1930     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1931   }
 1932 }
 1933 
 1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1935 {
 1936   return MachNode::size(ra_); // too many variables; just compute it
 1937                               // the hard way
 1938 }
 1939 
 1940 int MachPrologNode::reloc() const
 1941 {
 1942   return 0;
 1943 }
 1944 
 1945 //=============================================================================
 1946 
 1947 #ifndef PRODUCT
 1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1949   Compile* C = ra_->C;
 1950   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1951 
 1952   st->print("# pop frame %d\n\t",framesize);
 1953 
 1954   if (framesize == 0) {
 1955     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1956   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1957     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1958     st->print("add  sp, sp, #%d\n\t", framesize);
 1959   } else {
 1960     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, rscratch1\n\t");
 1962     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1963   }
 1964 
 1965   if (do_polling() && C->is_method_compilation()) {
 1966     st->print("# test polling word\n\t");
 1967     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1968     st->print("cmp  sp, rscratch1\n\t");
 1969     st->print("bhi #slow_path");
 1970   }
 1971 }
 1972 #endif
 1973 
 1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1975   Compile* C = ra_->C;
 1976   C2_MacroAssembler _masm(&cbuf);
 1977   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1978 
 1979   __ remove_frame(framesize);
 1980 
 1981   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1982     __ reserved_stack_check();
 1983   }
 1984 
 1985   if (do_polling() && C->is_method_compilation()) {
 1986     Label dummy_label;
 1987     Label* code_stub = &dummy_label;
 1988     if (!C->output()->in_scratch_emit_size()) {
 1989       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1990     }
 1991     __ relocate(relocInfo::poll_return_type);
 1992     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1993   }
 1994 }
 1995 
 1996 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1997   // Variable size. Determine dynamically.
 1998   return MachNode::size(ra_);
 1999 }
 2000 
 2001 int MachEpilogNode::reloc() const {
 2002   // Return number of relocatable values contained in this instruction.
 2003   return 1; // 1 for polling page.
 2004 }
 2005 
 2006 const Pipeline * MachEpilogNode::pipeline() const {
 2007   return MachNode::pipeline_class();
 2008 }
 2009 
 2010 //=============================================================================
 2011 
 2012 // Figure out which register class each belongs in: rc_int, rc_float or
 2013 // rc_stack.
 2014 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2015 
 2016 static enum RC rc_class(OptoReg::Name reg) {
 2017 
 2018   if (reg == OptoReg::Bad) {
 2019     return rc_bad;
 2020   }
 2021 
 2022   // we have 32 int registers * 2 halves
 2023   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2024 
 2025   if (reg < slots_of_int_registers) {
 2026     return rc_int;
 2027   }
 2028 
 2029   // we have 32 float register * 8 halves
 2030   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2031   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2032     return rc_float;
 2033   }
 2034 
 2035   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2036   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2037     return rc_predicate;
 2038   }
 2039 
 2040   // Between predicate regs & stack is the flags.
 2041   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2042 
 2043   return rc_stack;
 2044 }
 2045 
 2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2047   Compile* C = ra_->C;
 2048 
 2049   // Get registers to move.
 2050   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2051   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2052   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2053   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2054 
 2055   enum RC src_hi_rc = rc_class(src_hi);
 2056   enum RC src_lo_rc = rc_class(src_lo);
 2057   enum RC dst_hi_rc = rc_class(dst_hi);
 2058   enum RC dst_lo_rc = rc_class(dst_lo);
 2059 
 2060   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2061 
 2062   if (src_hi != OptoReg::Bad) {
 2063     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2064            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2065            "expected aligned-adjacent pairs");
 2066   }
 2067 
 2068   if (src_lo == dst_lo && src_hi == dst_hi) {
 2069     return 0;            // Self copy, no move.
 2070   }
 2071 
 2072   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2073               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2074   int src_offset = ra_->reg2offset(src_lo);
 2075   int dst_offset = ra_->reg2offset(dst_lo);
 2076 
 2077   if (bottom_type()->isa_vect() != NULL) {
 2078     uint ireg = ideal_reg();
 2079     if (ireg == Op_VecA && cbuf) {
 2080       C2_MacroAssembler _masm(cbuf);
 2081       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2082       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2083         // stack->stack
 2084         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2085                                                 sve_vector_reg_size_in_bytes);
 2086       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2087         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2088                             sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2090         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2091                               sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2093         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2094                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2095                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2096       } else {
 2097         ShouldNotReachHere();
 2098       }
 2099     } else if (cbuf) {
 2100       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2101       C2_MacroAssembler _masm(cbuf);
 2102       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2103       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2104         // stack->stack
 2105         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2106         if (ireg == Op_VecD) {
 2107           __ unspill(rscratch1, true, src_offset);
 2108           __ spill(rscratch1, true, dst_offset);
 2109         } else {
 2110           __ spill_copy128(src_offset, dst_offset);
 2111         }
 2112       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2113         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2114                ireg == Op_VecD ? __ T8B : __ T16B,
 2115                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2116       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2117         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2118                  ireg == Op_VecD ? __ D : __ Q,
 2119                  ra_->reg2offset(dst_lo));
 2120       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2121         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2122                    ireg == Op_VecD ? __ D : __ Q,
 2123                    ra_->reg2offset(src_lo));
 2124       } else {
 2125         ShouldNotReachHere();
 2126       }
 2127     }
 2128   } else if (cbuf) {
 2129     C2_MacroAssembler _masm(cbuf);
 2130     switch (src_lo_rc) {
 2131     case rc_int:
 2132       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2133         if (is64) {
 2134             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2135                    as_Register(Matcher::_regEncode[src_lo]));
 2136         } else {
 2137             C2_MacroAssembler _masm(cbuf);
 2138             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2139                     as_Register(Matcher::_regEncode[src_lo]));
 2140         }
 2141       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2142         if (is64) {
 2143             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2144                      as_Register(Matcher::_regEncode[src_lo]));
 2145         } else {
 2146             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2147                      as_Register(Matcher::_regEncode[src_lo]));
 2148         }
 2149       } else {                    // gpr --> stack spill
 2150         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2151         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2152       }
 2153       break;
 2154     case rc_float:
 2155       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2156         if (is64) {
 2157             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2158                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2159         } else {
 2160             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2161                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2162         }
 2163       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2164         if (is64) {
 2165             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2166                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2167         } else {
 2168             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         }
 2171       } else {                    // fpr --> stack spill
 2172         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2173         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2174                  is64 ? __ D : __ S, dst_offset);
 2175       }
 2176       break;
 2177     case rc_stack:
 2178       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2179         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2180       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2181         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2182                    is64 ? __ D : __ S, src_offset);
 2183       } else {                    // stack --> stack copy
 2184         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2185         __ unspill(rscratch1, is64, src_offset);
 2186         __ spill(rscratch1, is64, dst_offset);
 2187       }
 2188       break;
 2189     default:
 2190       assert(false, "bad rc_class for spill");
 2191       ShouldNotReachHere();
 2192     }
 2193   }
 2194 
 2195   if (st) {
 2196     st->print("spill ");
 2197     if (src_lo_rc == rc_stack) {
 2198       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2199     } else {
 2200       st->print("%s -> ", Matcher::regName[src_lo]);
 2201     }
 2202     if (dst_lo_rc == rc_stack) {
 2203       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2204     } else {
 2205       st->print("%s", Matcher::regName[dst_lo]);
 2206     }
 2207     if (bottom_type()->isa_vect() != NULL) {
 2208       int vsize = 0;
 2209       switch (ideal_reg()) {
 2210       case Op_VecD:
 2211         vsize = 64;
 2212         break;
 2213       case Op_VecX:
 2214         vsize = 128;
 2215         break;
 2216       case Op_VecA:
 2217         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2218         break;
 2219       default:
 2220         assert(false, "bad register type for spill");
 2221         ShouldNotReachHere();
 2222       }
 2223       st->print("\t# vector spill size = %d", vsize);
 2224     } else {
 2225       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2226     }
 2227   }
 2228 
 2229   return 0;
 2230 
 2231 }
 2232 
 2233 #ifndef PRODUCT
 2234 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2235   if (!ra_)
 2236     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2237   else
 2238     implementation(NULL, ra_, false, st);
 2239 }
 2240 #endif
 2241 
 2242 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2243   implementation(&cbuf, ra_, false, NULL);
 2244 }
 2245 
 2246 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2247   return MachNode::size(ra_);
 2248 }
 2249 
 2250 //=============================================================================
 2251 
 2252 #ifndef PRODUCT
 2253 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2254   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2255   int reg = ra_->get_reg_first(this);
 2256   st->print("add %s, rsp, #%d]\t# box lock",
 2257             Matcher::regName[reg], offset);
 2258 }
 2259 #endif
 2260 
 2261 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2262   C2_MacroAssembler _masm(&cbuf);
 2263 
 2264   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2265   int reg    = ra_->get_encode(this);
 2266 
 2267   // This add will handle any 24-bit signed offset. 24 bits allows an
 2268   // 8 megabyte stack frame.
 2269   __ add(as_Register(reg), sp, offset);
 2270 }
 2271 
 2272 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2273   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2274   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2275 
 2276   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2277     return NativeInstruction::instruction_size;
 2278   } else {
 2279     return 2 * NativeInstruction::instruction_size;
 2280   }
 2281 }
 2282 
 2283 //=============================================================================
 2284 
 2285 #ifndef PRODUCT
 2286 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2287 {
 2288   st->print_cr("# MachUEPNode");
 2289   if (UseCompressedClassPointers) {
 2290     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2291     if (CompressedKlassPointers::shift() != 0) {
 2292       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2293     }
 2294   } else {
 2295    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2296   }
 2297   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2298   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2299 }
 2300 #endif
 2301 
 2302 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2303 {
 2304   // This is the unverified entry point.
 2305   C2_MacroAssembler _masm(&cbuf);
 2306 
 2307   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2308   Label skip;
 2309   // TODO
 2310   // can we avoid this skip and still use a reloc?
 2311   __ br(Assembler::EQ, skip);
 2312   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2313   __ bind(skip);
 2314 }
 2315 
 2316 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2317 {
 2318   return MachNode::size(ra_);
 2319 }
 2320 
 2321 // REQUIRED EMIT CODE
 2322 
 2323 //=============================================================================
 2324 
 2325 // Emit exception handler code.
 2326 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2327 {
 2328   // mov rscratch1 #exception_blob_entry_point
 2329   // br rscratch1
 2330   // Note that the code buffer's insts_mark is always relative to insts.
 2331   // That's why we must use the macroassembler to generate a handler.
 2332   C2_MacroAssembler _masm(&cbuf);
 2333   address base = __ start_a_stub(size_exception_handler());
 2334   if (base == NULL) {
 2335     ciEnv::current()->record_failure("CodeCache is full");
 2336     return 0;  // CodeBuffer::expand failed
 2337   }
 2338   int offset = __ offset();
 2339   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2340   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2341   __ end_a_stub();
 2342   return offset;
 2343 }
 2344 
 2345 // Emit deopt handler code.
 2346 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2347 {
 2348   // Note that the code buffer's insts_mark is always relative to insts.
 2349   // That's why we must use the macroassembler to generate a handler.
 2350   C2_MacroAssembler _masm(&cbuf);
 2351   address base = __ start_a_stub(size_deopt_handler());
 2352   if (base == NULL) {
 2353     ciEnv::current()->record_failure("CodeCache is full");
 2354     return 0;  // CodeBuffer::expand failed
 2355   }
 2356   int offset = __ offset();
 2357 
 2358   __ adr(lr, __ pc());
 2359   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2360 
 2361   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
 2362   __ end_a_stub();
 2363   return offset;
 2364 }
 2365 
 2366 // REQUIRED MATCHER CODE
 2367 
 2368 //=============================================================================
 2369 
 2370 const bool Matcher::match_rule_supported(int opcode) {
 2371   if (!has_match_rule(opcode))
 2372     return false;
 2373 
 2374   bool ret_value = true;
 2375   switch (opcode) {
 2376     case Op_CacheWB:
 2377     case Op_CacheWBPreSync:
 2378     case Op_CacheWBPostSync:
 2379       if (!VM_Version::supports_data_cache_line_flush()) {
 2380         ret_value = false;
 2381       }
 2382       break;
 2383   }
 2384 
 2385   return ret_value; // Per default match rules are supported.
 2386 }
 2387 
 2388 // Identify extra cases that we might want to provide match rules for vector nodes and
 2389 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2390 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2391   if (!match_rule_supported(opcode)) {
 2392     return false;
 2393   }
 2394   int bit_size = vlen * type2aelembytes(bt) * 8;
 2395   if (UseSVE == 0 && bit_size > 128) {
 2396     return false;
 2397   }
 2398   if (UseSVE > 0) {
 2399     return op_sve_supported(opcode, vlen, bt);
 2400   } else { // NEON
 2401     // Special cases
 2402     switch (opcode) {
 2403     case Op_VectorMaskCmp:
 2404     // We don't have VectorReinterpret with bit_size less than 64 support for
 2405     // now, even for byte type. To be refined with fully VectorCast support.
 2406     case Op_VectorReinterpret:
 2407       if (vlen < 2 || bit_size < 64) {
 2408         return false;
 2409       }
 2410       break;
 2411     case Op_MulAddVS2VI:
 2412       if (bit_size < 128) {
 2413         return false;
 2414       }
 2415       break;
 2416     case Op_MulVL:
 2417       return false;
 2418     case Op_VectorLoadShuffle:
 2419     case Op_VectorRearrange:
 2420       if (vlen < 4) {
 2421         return false;
 2422       }
 2423       break;
 2424     // Some types of VectorCast are not implemented for now.
 2425     case Op_VectorCastI2X:
 2426       if (bt == T_BYTE) {
 2427         return false;
 2428       }
 2429       break;
 2430     case Op_VectorCastS2X:
 2431       if (vlen < 4 || bit_size < 64) {
 2432         return false;
 2433       }
 2434       break;
 2435     case Op_VectorCastF2X:
 2436     case Op_VectorCastD2X:
 2437       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2438         return false;
 2439       }
 2440       break;
 2441     case Op_LoadVectorGather:
 2442     case Op_StoreVectorScatter:
 2443       return false;
 2444     default:
 2445       break;
 2446     }
 2447   }
 2448   return vector_size_supported(bt, vlen);
 2449 }
 2450 
 2451 const RegMask* Matcher::predicate_reg_mask(void) {
 2452   return &_PR_REG_mask;
 2453 }
 2454 
 2455 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2456   return new TypeVectMask(elemTy, length);
 2457 }
 2458 
 2459 // Vector calling convention not yet implemented.
 2460 const bool Matcher::supports_vector_calling_convention(void) {
 2461   return false;
 2462 }
 2463 
 2464 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2465   Unimplemented();
 2466   return OptoRegPair(0, 0);
 2467 }
 2468 
 2469 // Is this branch offset short enough that a short branch can be used?
 2470 //
 2471 // NOTE: If the platform does not provide any short branch variants, then
 2472 //       this method should return false for offset 0.
 2473 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2474   // The passed offset is relative to address of the branch.
 2475 
 2476   return (-32768 <= offset && offset < 32768);
 2477 }
 2478 
 2479 // Vector width in bytes.
 2480 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2481   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2482   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2483   // Minimum 2 values in vector
 2484   if (size < 2*type2aelembytes(bt)) size = 0;
 2485   // But never < 4
 2486   if (size < 4) size = 0;
 2487   return size;
 2488 }
 2489 
 2490 // Limits on vector size (number of elements) loaded into vector.
 2491 const int Matcher::max_vector_size(const BasicType bt) {
 2492   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2493 }
 2494 
 2495 const int Matcher::min_vector_size(const BasicType bt) {
 2496   int max_size = max_vector_size(bt);
 2497   // Limit the min vector size to 8 bytes.
 2498   int size = 8 / type2aelembytes(bt);
 2499   if (bt == T_BYTE) {
 2500     // To support vector api shuffle/rearrange.
 2501     size = 4;
 2502   } else if (bt == T_BOOLEAN) {
 2503     // To support vector api load/store mask.
 2504     size = 2;
 2505   }
 2506   if (size < 2) size = 2;
 2507   return MIN2(size, max_size);
 2508 }
 2509 
 2510 // Actual max scalable vector register length.
 2511 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2512   return Matcher::max_vector_size(bt);
 2513 }
 2514 
 2515 // Vector ideal reg.
 2516 const uint Matcher::vector_ideal_reg(int len) {
 2517   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2518     return Op_VecA;
 2519   }
 2520   switch(len) {
 2521     // For 16-bit/32-bit mask vector, reuse VecD.
 2522     case  2:
 2523     case  4:
 2524     case  8: return Op_VecD;
 2525     case 16: return Op_VecX;
 2526   }
 2527   ShouldNotReachHere();
 2528   return 0;
 2529 }
 2530 
 2531 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2532   ShouldNotReachHere(); // generic vector operands not supported
 2533   return NULL;
 2534 }
 2535 
 2536 bool Matcher::is_reg2reg_move(MachNode* m) {
 2537   ShouldNotReachHere();  // generic vector operands not supported
 2538   return false;
 2539 }
 2540 
 2541 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2542   ShouldNotReachHere();  // generic vector operands not supported
 2543   return false;
 2544 }
 2545 
 2546 // Return whether or not this register is ever used as an argument.
 2547 // This function is used on startup to build the trampoline stubs in
 2548 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2549 // call in the trampoline, and arguments in those registers not be
 2550 // available to the callee.
 2551 bool Matcher::can_be_java_arg(int reg)
 2552 {
 2553   return
 2554     reg ==  R0_num || reg == R0_H_num ||
 2555     reg ==  R1_num || reg == R1_H_num ||
 2556     reg ==  R2_num || reg == R2_H_num ||
 2557     reg ==  R3_num || reg == R3_H_num ||
 2558     reg ==  R4_num || reg == R4_H_num ||
 2559     reg ==  R5_num || reg == R5_H_num ||
 2560     reg ==  R6_num || reg == R6_H_num ||
 2561     reg ==  R7_num || reg == R7_H_num ||
 2562     reg ==  V0_num || reg == V0_H_num ||
 2563     reg ==  V1_num || reg == V1_H_num ||
 2564     reg ==  V2_num || reg == V2_H_num ||
 2565     reg ==  V3_num || reg == V3_H_num ||
 2566     reg ==  V4_num || reg == V4_H_num ||
 2567     reg ==  V5_num || reg == V5_H_num ||
 2568     reg ==  V6_num || reg == V6_H_num ||
 2569     reg ==  V7_num || reg == V7_H_num;
 2570 }
 2571 
 2572 bool Matcher::is_spillable_arg(int reg)
 2573 {
 2574   return can_be_java_arg(reg);
 2575 }
 2576 
 2577 uint Matcher::int_pressure_limit()
 2578 {
 2579   // JDK-8183543: When taking the number of available registers as int
 2580   // register pressure threshold, the jtreg test:
 2581   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2582   // failed due to C2 compilation failure with
 2583   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2584   //
 2585   // A derived pointer is live at CallNode and then is flagged by RA
 2586   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2587   // derived pointers and lastly fail to spill after reaching maximum
 2588   // number of iterations. Lowering the default pressure threshold to
 2589   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2590   // a high register pressure area of the code so that split_DEF can
 2591   // generate DefinitionSpillCopy for the derived pointer.
 2592   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2593   if (!PreserveFramePointer) {
 2594     // When PreserveFramePointer is off, frame pointer is allocatable,
 2595     // but different from other SOC registers, it is excluded from
 2596     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2597     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2598     // See check_pressure_at_fatproj().
 2599     default_int_pressure_threshold--;
 2600   }
 2601   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2602 }
 2603 
 2604 uint Matcher::float_pressure_limit()
 2605 {
 2606   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2607   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2608 }
 2609 
 2610 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2611   return false;
 2612 }
 2613 
 2614 RegMask Matcher::divI_proj_mask() {
 2615   ShouldNotReachHere();
 2616   return RegMask();
 2617 }
 2618 
 2619 // Register for MODI projection of divmodI.
 2620 RegMask Matcher::modI_proj_mask() {
 2621   ShouldNotReachHere();
 2622   return RegMask();
 2623 }
 2624 
 2625 // Register for DIVL projection of divmodL.
 2626 RegMask Matcher::divL_proj_mask() {
 2627   ShouldNotReachHere();
 2628   return RegMask();
 2629 }
 2630 
 2631 // Register for MODL projection of divmodL.
 2632 RegMask Matcher::modL_proj_mask() {
 2633   ShouldNotReachHere();
 2634   return RegMask();
 2635 }
 2636 
 2637 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2638   return FP_REG_mask();
 2639 }
 2640 
 2641 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2642   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2643     Node* u = addp->fast_out(i);
 2644     if (u->is_LoadStore()) {
 2645       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2646       // instructions) only take register indirect as an operand, so
 2647       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2648       // must fail.
 2649       return false;
 2650     }
 2651     if (u->is_Mem()) {
 2652       int opsize = u->as_Mem()->memory_size();
 2653       assert(opsize > 0, "unexpected memory operand size");
 2654       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2655         return false;
 2656       }
 2657     }
 2658   }
 2659   return true;
 2660 }
 2661 
 2662 // Should the matcher clone input 'm' of node 'n'?
 2663 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2664   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
 2665     mstack.push(m, Visit);           // m = ShiftCntV
 2666     return true;
 2667   }
 2668   return false;
 2669 }
 2670 
 2671 // Should the Matcher clone shifts on addressing modes, expecting them
 2672 // to be subsumed into complex addressing expressions or compute them
 2673 // into registers?
 2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2675   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2676     return true;
 2677   }
 2678 
 2679   Node *off = m->in(AddPNode::Offset);
 2680   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2681       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2682       // Are there other uses besides address expressions?
 2683       !is_visited(off)) {
 2684     address_visited.set(off->_idx); // Flag as address_visited
 2685     mstack.push(off->in(2), Visit);
 2686     Node *conv = off->in(1);
 2687     if (conv->Opcode() == Op_ConvI2L &&
 2688         // Are there other uses besides address expressions?
 2689         !is_visited(conv)) {
 2690       address_visited.set(conv->_idx); // Flag as address_visited
 2691       mstack.push(conv->in(1), Pre_Visit);
 2692     } else {
 2693       mstack.push(conv, Pre_Visit);
 2694     }
 2695     address_visited.test_set(m->_idx); // Flag as address_visited
 2696     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2697     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2698     return true;
 2699   } else if (off->Opcode() == Op_ConvI2L &&
 2700              // Are there other uses besides address expressions?
 2701              !is_visited(off)) {
 2702     address_visited.test_set(m->_idx); // Flag as address_visited
 2703     address_visited.set(off->_idx); // Flag as address_visited
 2704     mstack.push(off->in(1), Pre_Visit);
 2705     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2706     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2707     return true;
 2708   }
 2709   return false;
 2710 }
 2711 
 2712 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2713   C2_MacroAssembler _masm(&cbuf);                                       \
 2714   {                                                                     \
 2715     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2716     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2717     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2718     __ INSN(REG, as_Register(BASE));                                    \
 2719   }
 2720 
 2721 
 2722 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2723   {
 2724     Address::extend scale;
 2725 
 2726     // Hooboy, this is fugly.  We need a way to communicate to the
 2727     // encoder that the index needs to be sign extended, so we have to
 2728     // enumerate all the cases.
 2729     switch (opcode) {
 2730     case INDINDEXSCALEDI2L:
 2731     case INDINDEXSCALEDI2LN:
 2732     case INDINDEXI2L:
 2733     case INDINDEXI2LN:
 2734       scale = Address::sxtw(size);
 2735       break;
 2736     default:
 2737       scale = Address::lsl(size);
 2738     }
 2739 
 2740     if (index == -1) {
 2741       return Address(base, disp);
 2742     } else {
 2743       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2744       return Address(base, as_Register(index), scale);
 2745     }
 2746   }
 2747 
 2748 
 2749 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2750 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2751 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2752 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2753                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2754 
 2755   // Used for all non-volatile memory accesses.  The use of
 2756   // $mem->opcode() to discover whether this pattern uses sign-extended
 2757   // offsets is something of a kludge.
 2758   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2759                         Register reg, int opcode,
 2760                         Register base, int index, int scale, int disp,
 2761                         int size_in_memory)
 2762   {
 2763     Address addr = mem2address(opcode, base, index, scale, disp);
 2764     if (addr.getMode() == Address::base_plus_offset) {
 2765       /* If we get an out-of-range offset it is a bug in the compiler,
 2766          so we assert here. */
 2767       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2768              "c2 compiler bug");
 2769       /* Fix up any out-of-range offsets. */
 2770       assert_different_registers(rscratch1, base);
 2771       assert_different_registers(rscratch1, reg);
 2772       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2773     }
 2774     (masm.*insn)(reg, addr);
 2775   }
 2776 
 2777   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2778                         FloatRegister reg, int opcode,
 2779                         Register base, int index, int size, int disp,
 2780                         int size_in_memory)
 2781   {
 2782     Address::extend scale;
 2783 
 2784     switch (opcode) {
 2785     case INDINDEXSCALEDI2L:
 2786     case INDINDEXSCALEDI2LN:
 2787       scale = Address::sxtw(size);
 2788       break;
 2789     default:
 2790       scale = Address::lsl(size);
 2791     }
 2792 
 2793     if (index == -1) {
 2794       /* If we get an out-of-range offset it is a bug in the compiler,
 2795          so we assert here. */
 2796       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2797       /* Fix up any out-of-range offsets. */
 2798       assert_different_registers(rscratch1, base);
 2799       Address addr = Address(base, disp);
 2800       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2801       (masm.*insn)(reg, addr);
 2802     } else {
 2803       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2804       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2805     }
 2806   }
 2807 
 2808   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2809                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2810                         int opcode, Register base, int index, int size, int disp)
 2811   {
 2812     if (index == -1) {
 2813       (masm.*insn)(reg, T, Address(base, disp));
 2814     } else {
 2815       assert(disp == 0, "unsupported address mode");
 2816       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2817     }
 2818   }
 2819 
 2820 %}
 2821 
 2822 
 2823 
 2824 //----------ENCODING BLOCK-----------------------------------------------------
 2825 // This block specifies the encoding classes used by the compiler to
 2826 // output byte streams.  Encoding classes are parameterized macros
 2827 // used by Machine Instruction Nodes in order to generate the bit
 2828 // encoding of the instruction.  Operands specify their base encoding
 2829 // interface with the interface keyword.  There are currently
 2830 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2831 // COND_INTER.  REG_INTER causes an operand to generate a function
 2832 // which returns its register number when queried.  CONST_INTER causes
 2833 // an operand to generate a function which returns the value of the
 2834 // constant when queried.  MEMORY_INTER causes an operand to generate
 2835 // four functions which return the Base Register, the Index Register,
 2836 // the Scale Value, and the Offset Value of the operand when queried.
 2837 // COND_INTER causes an operand to generate six functions which return
 2838 // the encoding code (ie - encoding bits for the instruction)
 2839 // associated with each basic boolean condition for a conditional
 2840 // instruction.
 2841 //
 2842 // Instructions specify two basic values for encoding.  Again, a
 2843 // function is available to check if the constant displacement is an
 2844 // oop. They use the ins_encode keyword to specify their encoding
 2845 // classes (which must be a sequence of enc_class names, and their
 2846 // parameters, specified in the encoding block), and they use the
 2847 // opcode keyword to specify, in order, their primary, secondary, and
 2848 // tertiary opcode.  Only the opcode sections which a particular
 2849 // instruction needs for encoding need to be specified.
 2850 encode %{
 2851   // Build emit functions for each basic byte or larger field in the
 2852   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2853   // from C++ code in the enc_class source block.  Emit functions will
 2854   // live in the main source block for now.  In future, we can
 2855   // generalize this by adding a syntax that specifies the sizes of
 2856   // fields in an order, so that the adlc can build the emit functions
 2857   // automagically
 2858 
 2859   // catch all for unimplemented encodings
 2860   enc_class enc_unimplemented %{
 2861     C2_MacroAssembler _masm(&cbuf);
 2862     __ unimplemented("C2 catch all");
 2863   %}
 2864 
 2865   // BEGIN Non-volatile memory access
 2866 
 2867   // This encoding class is generated automatically from ad_encode.m4.
 2868   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2869   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2870     Register dst_reg = as_Register($dst$$reg);
 2871     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2872                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2873   %}
 2874 
 2875   // This encoding class is generated automatically from ad_encode.m4.
 2876   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2877   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2878     Register dst_reg = as_Register($dst$$reg);
 2879     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2880                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2881   %}
 2882 
 2883   // This encoding class is generated automatically from ad_encode.m4.
 2884   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2885   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2886     Register dst_reg = as_Register($dst$$reg);
 2887     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2888                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2889   %}
 2890 
 2891   // This encoding class is generated automatically from ad_encode.m4.
 2892   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2893   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2894     Register dst_reg = as_Register($dst$$reg);
 2895     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2896                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2897   %}
 2898 
 2899   // This encoding class is generated automatically from ad_encode.m4.
 2900   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2901   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2902     Register dst_reg = as_Register($dst$$reg);
 2903     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2904                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2905   %}
 2906 
 2907   // This encoding class is generated automatically from ad_encode.m4.
 2908   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2909   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2910     Register dst_reg = as_Register($dst$$reg);
 2911     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2912                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2913   %}
 2914 
 2915   // This encoding class is generated automatically from ad_encode.m4.
 2916   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2917   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2918     Register dst_reg = as_Register($dst$$reg);
 2919     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2921   %}
 2922 
 2923   // This encoding class is generated automatically from ad_encode.m4.
 2924   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2925   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2926     Register dst_reg = as_Register($dst$$reg);
 2927     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2928                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2929   %}
 2930 
 2931   // This encoding class is generated automatically from ad_encode.m4.
 2932   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2933   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2934     Register dst_reg = as_Register($dst$$reg);
 2935     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2936                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2937   %}
 2938 
 2939   // This encoding class is generated automatically from ad_encode.m4.
 2940   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2941   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2942     Register dst_reg = as_Register($dst$$reg);
 2943     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2945   %}
 2946 
 2947   // This encoding class is generated automatically from ad_encode.m4.
 2948   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2949   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2950     Register dst_reg = as_Register($dst$$reg);
 2951     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2952                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2953   %}
 2954 
 2955   // This encoding class is generated automatically from ad_encode.m4.
 2956   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2957   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2958     Register dst_reg = as_Register($dst$$reg);
 2959     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2960                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2961   %}
 2962 
 2963   // This encoding class is generated automatically from ad_encode.m4.
 2964   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2965   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2966     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2967     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2968                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2969   %}
 2970 
 2971   // This encoding class is generated automatically from ad_encode.m4.
 2972   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2973   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2974     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2975     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2976                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2977   %}
 2978 
 2979   // This encoding class is generated automatically from ad_encode.m4.
 2980   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2981   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2982     Register src_reg = as_Register($src$$reg);
 2983     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2984                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2985   %}
 2986 
 2987   // This encoding class is generated automatically from ad_encode.m4.
 2988   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2989   enc_class aarch64_enc_strb0(memory1 mem) %{
 2990     C2_MacroAssembler _masm(&cbuf);
 2991     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2992                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2993   %}
 2994 
 2995   // This encoding class is generated automatically from ad_encode.m4.
 2996   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2997   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2998     Register src_reg = as_Register($src$$reg);
 2999     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3000                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3001   %}
 3002 
 3003   // This encoding class is generated automatically from ad_encode.m4.
 3004   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3005   enc_class aarch64_enc_strh0(memory2 mem) %{
 3006     C2_MacroAssembler _masm(&cbuf);
 3007     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3008                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3009   %}
 3010 
 3011   // This encoding class is generated automatically from ad_encode.m4.
 3012   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3013   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3014     Register src_reg = as_Register($src$$reg);
 3015     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3016                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3017   %}
 3018 
 3019   // This encoding class is generated automatically from ad_encode.m4.
 3020   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3021   enc_class aarch64_enc_strw0(memory4 mem) %{
 3022     C2_MacroAssembler _masm(&cbuf);
 3023     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3024                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3025   %}
 3026 
 3027   // This encoding class is generated automatically from ad_encode.m4.
 3028   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3029   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3030     Register src_reg = as_Register($src$$reg);
 3031     // we sometimes get asked to store the stack pointer into the
 3032     // current thread -- we cannot do that directly on AArch64
 3033     if (src_reg == r31_sp) {
 3034       C2_MacroAssembler _masm(&cbuf);
 3035       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3036       __ mov(rscratch2, sp);
 3037       src_reg = rscratch2;
 3038     }
 3039     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3041   %}
 3042 
 3043   // This encoding class is generated automatically from ad_encode.m4.
 3044   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3045   enc_class aarch64_enc_str0(memory8 mem) %{
 3046     C2_MacroAssembler _masm(&cbuf);
 3047     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3048                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3049   %}
 3050 
 3051   // This encoding class is generated automatically from ad_encode.m4.
 3052   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3053   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3054     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3055     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3056                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3057   %}
 3058 
 3059   // This encoding class is generated automatically from ad_encode.m4.
 3060   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3061   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3062     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3063     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3064                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3065   %}
 3066 
 3067   // This encoding class is generated automatically from ad_encode.m4.
 3068   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3069   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3070       C2_MacroAssembler _masm(&cbuf);
 3071       __ membar(Assembler::StoreStore);
 3072       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3073                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3074   %}
 3075 
 3076   // END Non-volatile memory access
 3077 
 3078   // Vector loads and stores
 3079   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3080     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3081     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3082        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3083   %}
 3084 
 3085   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3086     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3087     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3088        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3089   %}
 3090 
 3091   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3092     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3093     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3094        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3095   %}
 3096 
 3097   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3098     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3099     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3100        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3101   %}
 3102 
 3103   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3104     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3105     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3106        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3107   %}
 3108 
 3109   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3110     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3111     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3112        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3113   %}
 3114 
 3115   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3116     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3117     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3118        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3122     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3123     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3124        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3125   %}
 3126 
 3127   // volatile loads and stores
 3128 
 3129   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3130     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3131                  rscratch1, stlrb);
 3132   %}
 3133 
 3134   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3135     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136                  rscratch1, stlrh);
 3137   %}
 3138 
 3139   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3140     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3141                  rscratch1, stlrw);
 3142   %}
 3143 
 3144 
 3145   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3146     Register dst_reg = as_Register($dst$$reg);
 3147     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3148              rscratch1, ldarb);
 3149     __ sxtbw(dst_reg, dst_reg);
 3150   %}
 3151 
 3152   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3153     Register dst_reg = as_Register($dst$$reg);
 3154     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3155              rscratch1, ldarb);
 3156     __ sxtb(dst_reg, dst_reg);
 3157   %}
 3158 
 3159   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3160     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3161              rscratch1, ldarb);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3165     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3166              rscratch1, ldarb);
 3167   %}
 3168 
 3169   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3170     Register dst_reg = as_Register($dst$$reg);
 3171     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3172              rscratch1, ldarh);
 3173     __ sxthw(dst_reg, dst_reg);
 3174   %}
 3175 
 3176   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3177     Register dst_reg = as_Register($dst$$reg);
 3178     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3179              rscratch1, ldarh);
 3180     __ sxth(dst_reg, dst_reg);
 3181   %}
 3182 
 3183   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3184     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3185              rscratch1, ldarh);
 3186   %}
 3187 
 3188   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3189     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3190              rscratch1, ldarh);
 3191   %}
 3192 
 3193   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3194     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3195              rscratch1, ldarw);
 3196   %}
 3197 
 3198   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3199     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3200              rscratch1, ldarw);
 3201   %}
 3202 
 3203   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3204     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3205              rscratch1, ldar);
 3206   %}
 3207 
 3208   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3209     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3210              rscratch1, ldarw);
 3211     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3212   %}
 3213 
 3214   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3215     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3216              rscratch1, ldar);
 3217     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3218   %}
 3219 
 3220   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3221     Register src_reg = as_Register($src$$reg);
 3222     // we sometimes get asked to store the stack pointer into the
 3223     // current thread -- we cannot do that directly on AArch64
 3224     if (src_reg == r31_sp) {
 3225       C2_MacroAssembler _masm(&cbuf);
 3226       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3227       __ mov(rscratch2, sp);
 3228       src_reg = rscratch2;
 3229     }
 3230     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3231                  rscratch1, stlr);
 3232   %}
 3233 
 3234   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3235     {
 3236       C2_MacroAssembler _masm(&cbuf);
 3237       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3238       __ fmovs(rscratch2, src_reg);
 3239     }
 3240     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3241                  rscratch1, stlrw);
 3242   %}
 3243 
 3244   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3245     {
 3246       C2_MacroAssembler _masm(&cbuf);
 3247       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3248       __ fmovd(rscratch2, src_reg);
 3249     }
 3250     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3251                  rscratch1, stlr);
 3252   %}
 3253 
 3254   // synchronized read/update encodings
 3255 
 3256   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3257     C2_MacroAssembler _masm(&cbuf);
 3258     Register dst_reg = as_Register($dst$$reg);
 3259     Register base = as_Register($mem$$base);
 3260     int index = $mem$$index;
 3261     int scale = $mem$$scale;
 3262     int disp = $mem$$disp;
 3263     if (index == -1) {
 3264        if (disp != 0) {
 3265         __ lea(rscratch1, Address(base, disp));
 3266         __ ldaxr(dst_reg, rscratch1);
 3267       } else {
 3268         // TODO
 3269         // should we ever get anything other than this case?
 3270         __ ldaxr(dst_reg, base);
 3271       }
 3272     } else {
 3273       Register index_reg = as_Register(index);
 3274       if (disp == 0) {
 3275         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3276         __ ldaxr(dst_reg, rscratch1);
 3277       } else {
 3278         __ lea(rscratch1, Address(base, disp));
 3279         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3280         __ ldaxr(dst_reg, rscratch1);
 3281       }
 3282     }
 3283   %}
 3284 
 3285   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3286     C2_MacroAssembler _masm(&cbuf);
 3287     Register src_reg = as_Register($src$$reg);
 3288     Register base = as_Register($mem$$base);
 3289     int index = $mem$$index;
 3290     int scale = $mem$$scale;
 3291     int disp = $mem$$disp;
 3292     if (index == -1) {
 3293        if (disp != 0) {
 3294         __ lea(rscratch2, Address(base, disp));
 3295         __ stlxr(rscratch1, src_reg, rscratch2);
 3296       } else {
 3297         // TODO
 3298         // should we ever get anything other than this case?
 3299         __ stlxr(rscratch1, src_reg, base);
 3300       }
 3301     } else {
 3302       Register index_reg = as_Register(index);
 3303       if (disp == 0) {
 3304         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3305         __ stlxr(rscratch1, src_reg, rscratch2);
 3306       } else {
 3307         __ lea(rscratch2, Address(base, disp));
 3308         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3309         __ stlxr(rscratch1, src_reg, rscratch2);
 3310       }
 3311     }
 3312     __ cmpw(rscratch1, zr);
 3313   %}
 3314 
 3315   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3316     C2_MacroAssembler _masm(&cbuf);
 3317     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3318     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3319                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3320                /*weak*/ false, noreg);
 3321   %}
 3322 
 3323   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3324     C2_MacroAssembler _masm(&cbuf);
 3325     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3326     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3327                Assembler::word, /*acquire*/ false, /*release*/ true,
 3328                /*weak*/ false, noreg);
 3329   %}
 3330 
 3331   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3332     C2_MacroAssembler _masm(&cbuf);
 3333     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3334     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3335                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3336                /*weak*/ false, noreg);
 3337   %}
 3338 
 3339   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3340     C2_MacroAssembler _masm(&cbuf);
 3341     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3342     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3343                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3344                /*weak*/ false, noreg);
 3345   %}
 3346 
 3347 
 3348   // The only difference between aarch64_enc_cmpxchg and
 3349   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3350   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3351   // lock.
 3352   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3353     C2_MacroAssembler _masm(&cbuf);
 3354     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3355     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3356                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3357                /*weak*/ false, noreg);
 3358   %}
 3359 
 3360   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3361     C2_MacroAssembler _masm(&cbuf);
 3362     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3363     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3364                Assembler::word, /*acquire*/ true, /*release*/ true,
 3365                /*weak*/ false, noreg);
 3366   %}
 3367 
 3368   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3369     C2_MacroAssembler _masm(&cbuf);
 3370     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3371     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3372                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3373                /*weak*/ false, noreg);
 3374   %}
 3375 
 3376   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3377     C2_MacroAssembler _masm(&cbuf);
 3378     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3379     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3380                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3381                /*weak*/ false, noreg);
 3382   %}
 3383 
 3384   // auxiliary used for CompareAndSwapX to set result register
 3385   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3386     C2_MacroAssembler _masm(&cbuf);
 3387     Register res_reg = as_Register($res$$reg);
 3388     __ cset(res_reg, Assembler::EQ);
 3389   %}
 3390 
 3391   // prefetch encodings
 3392 
 3393   enc_class aarch64_enc_prefetchw(memory mem) %{
 3394     C2_MacroAssembler _masm(&cbuf);
 3395     Register base = as_Register($mem$$base);
 3396     int index = $mem$$index;
 3397     int scale = $mem$$scale;
 3398     int disp = $mem$$disp;
 3399     if (index == -1) {
 3400       __ prfm(Address(base, disp), PSTL1KEEP);
 3401     } else {
 3402       Register index_reg = as_Register(index);
 3403       if (disp == 0) {
 3404         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3405       } else {
 3406         __ lea(rscratch1, Address(base, disp));
 3407 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3408       }
 3409     }
 3410   %}
 3411 
 3412   /// mov envcodings
 3413 
 3414   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3415     C2_MacroAssembler _masm(&cbuf);
 3416     uint32_t con = (uint32_t)$src$$constant;
 3417     Register dst_reg = as_Register($dst$$reg);
 3418     if (con == 0) {
 3419       __ movw(dst_reg, zr);
 3420     } else {
 3421       __ movw(dst_reg, con);
 3422     }
 3423   %}
 3424 
 3425   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3426     C2_MacroAssembler _masm(&cbuf);
 3427     Register dst_reg = as_Register($dst$$reg);
 3428     uint64_t con = (uint64_t)$src$$constant;
 3429     if (con == 0) {
 3430       __ mov(dst_reg, zr);
 3431     } else {
 3432       __ mov(dst_reg, con);
 3433     }
 3434   %}
 3435 
 3436   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3437     C2_MacroAssembler _masm(&cbuf);
 3438     Register dst_reg = as_Register($dst$$reg);
 3439     address con = (address)$src$$constant;
 3440     if (con == NULL || con == (address)1) {
 3441       ShouldNotReachHere();
 3442     } else {
 3443       relocInfo::relocType rtype = $src->constant_reloc();
 3444       if (rtype == relocInfo::oop_type) {
 3445         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3446       } else if (rtype == relocInfo::metadata_type) {
 3447         __ mov_metadata(dst_reg, (Metadata*)con);
 3448       } else {
 3449         assert(rtype == relocInfo::none, "unexpected reloc type");
 3450         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3451           __ mov(dst_reg, con);
 3452         } else {
 3453           uint64_t offset;
 3454           __ adrp(dst_reg, con, offset);
 3455           __ add(dst_reg, dst_reg, offset);
 3456         }
 3457       }
 3458     }
 3459   %}
 3460 
 3461   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3462     C2_MacroAssembler _masm(&cbuf);
 3463     Register dst_reg = as_Register($dst$$reg);
 3464     __ mov(dst_reg, zr);
 3465   %}
 3466 
 3467   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3468     C2_MacroAssembler _masm(&cbuf);
 3469     Register dst_reg = as_Register($dst$$reg);
 3470     __ mov(dst_reg, (uint64_t)1);
 3471   %}
 3472 
 3473   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3474     C2_MacroAssembler _masm(&cbuf);
 3475     __ load_byte_map_base($dst$$Register);
 3476   %}
 3477 
 3478   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480     Register dst_reg = as_Register($dst$$reg);
 3481     address con = (address)$src$$constant;
 3482     if (con == NULL) {
 3483       ShouldNotReachHere();
 3484     } else {
 3485       relocInfo::relocType rtype = $src->constant_reloc();
 3486       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3487       __ set_narrow_oop(dst_reg, (jobject)con);
 3488     }
 3489   %}
 3490 
 3491   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3492     C2_MacroAssembler _masm(&cbuf);
 3493     Register dst_reg = as_Register($dst$$reg);
 3494     __ mov(dst_reg, zr);
 3495   %}
 3496 
 3497   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3498     C2_MacroAssembler _masm(&cbuf);
 3499     Register dst_reg = as_Register($dst$$reg);
 3500     address con = (address)$src$$constant;
 3501     if (con == NULL) {
 3502       ShouldNotReachHere();
 3503     } else {
 3504       relocInfo::relocType rtype = $src->constant_reloc();
 3505       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3506       __ set_narrow_klass(dst_reg, (Klass *)con);
 3507     }
 3508   %}
 3509 
 3510   // arithmetic encodings
 3511 
 3512   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3513     C2_MacroAssembler _masm(&cbuf);
 3514     Register dst_reg = as_Register($dst$$reg);
 3515     Register src_reg = as_Register($src1$$reg);
 3516     int32_t con = (int32_t)$src2$$constant;
 3517     // add has primary == 0, subtract has primary == 1
 3518     if ($primary) { con = -con; }
 3519     if (con < 0) {
 3520       __ subw(dst_reg, src_reg, -con);
 3521     } else {
 3522       __ addw(dst_reg, src_reg, con);
 3523     }
 3524   %}
 3525 
 3526   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3527     C2_MacroAssembler _masm(&cbuf);
 3528     Register dst_reg = as_Register($dst$$reg);
 3529     Register src_reg = as_Register($src1$$reg);
 3530     int32_t con = (int32_t)$src2$$constant;
 3531     // add has primary == 0, subtract has primary == 1
 3532     if ($primary) { con = -con; }
 3533     if (con < 0) {
 3534       __ sub(dst_reg, src_reg, -con);
 3535     } else {
 3536       __ add(dst_reg, src_reg, con);
 3537     }
 3538   %}
 3539 
 3540   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3541     C2_MacroAssembler _masm(&cbuf);
 3542    Register dst_reg = as_Register($dst$$reg);
 3543    Register src1_reg = as_Register($src1$$reg);
 3544    Register src2_reg = as_Register($src2$$reg);
 3545     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3546   %}
 3547 
 3548   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3549     C2_MacroAssembler _masm(&cbuf);
 3550    Register dst_reg = as_Register($dst$$reg);
 3551    Register src1_reg = as_Register($src1$$reg);
 3552    Register src2_reg = as_Register($src2$$reg);
 3553     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3554   %}
 3555 
 3556   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3557     C2_MacroAssembler _masm(&cbuf);
 3558    Register dst_reg = as_Register($dst$$reg);
 3559    Register src1_reg = as_Register($src1$$reg);
 3560    Register src2_reg = as_Register($src2$$reg);
 3561     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3562   %}
 3563 
 3564   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3565     C2_MacroAssembler _masm(&cbuf);
 3566    Register dst_reg = as_Register($dst$$reg);
 3567    Register src1_reg = as_Register($src1$$reg);
 3568    Register src2_reg = as_Register($src2$$reg);
 3569     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3570   %}
 3571 
 3572   // compare instruction encodings
 3573 
 3574   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3575     C2_MacroAssembler _masm(&cbuf);
 3576     Register reg1 = as_Register($src1$$reg);
 3577     Register reg2 = as_Register($src2$$reg);
 3578     __ cmpw(reg1, reg2);
 3579   %}
 3580 
 3581   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3582     C2_MacroAssembler _masm(&cbuf);
 3583     Register reg = as_Register($src1$$reg);
 3584     int32_t val = $src2$$constant;
 3585     if (val >= 0) {
 3586       __ subsw(zr, reg, val);
 3587     } else {
 3588       __ addsw(zr, reg, -val);
 3589     }
 3590   %}
 3591 
 3592   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3593     C2_MacroAssembler _masm(&cbuf);
 3594     Register reg1 = as_Register($src1$$reg);
 3595     uint32_t val = (uint32_t)$src2$$constant;
 3596     __ movw(rscratch1, val);
 3597     __ cmpw(reg1, rscratch1);
 3598   %}
 3599 
 3600   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3601     C2_MacroAssembler _masm(&cbuf);
 3602     Register reg1 = as_Register($src1$$reg);
 3603     Register reg2 = as_Register($src2$$reg);
 3604     __ cmp(reg1, reg2);
 3605   %}
 3606 
 3607   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3608     C2_MacroAssembler _masm(&cbuf);
 3609     Register reg = as_Register($src1$$reg);
 3610     int64_t val = $src2$$constant;
 3611     if (val >= 0) {
 3612       __ subs(zr, reg, val);
 3613     } else if (val != -val) {
 3614       __ adds(zr, reg, -val);
 3615     } else {
 3616     // aargh, Long.MIN_VALUE is a special case
 3617       __ orr(rscratch1, zr, (uint64_t)val);
 3618       __ subs(zr, reg, rscratch1);
 3619     }
 3620   %}
 3621 
 3622   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3623     C2_MacroAssembler _masm(&cbuf);
 3624     Register reg1 = as_Register($src1$$reg);
 3625     uint64_t val = (uint64_t)$src2$$constant;
 3626     __ mov(rscratch1, val);
 3627     __ cmp(reg1, rscratch1);
 3628   %}
 3629 
 3630   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3631     C2_MacroAssembler _masm(&cbuf);
 3632     Register reg1 = as_Register($src1$$reg);
 3633     Register reg2 = as_Register($src2$$reg);
 3634     __ cmp(reg1, reg2);
 3635   %}
 3636 
 3637   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3638     C2_MacroAssembler _masm(&cbuf);
 3639     Register reg1 = as_Register($src1$$reg);
 3640     Register reg2 = as_Register($src2$$reg);
 3641     __ cmpw(reg1, reg2);
 3642   %}
 3643 
 3644   enc_class aarch64_enc_testp(iRegP src) %{
 3645     C2_MacroAssembler _masm(&cbuf);
 3646     Register reg = as_Register($src$$reg);
 3647     __ cmp(reg, zr);
 3648   %}
 3649 
 3650   enc_class aarch64_enc_testn(iRegN src) %{
 3651     C2_MacroAssembler _masm(&cbuf);
 3652     Register reg = as_Register($src$$reg);
 3653     __ cmpw(reg, zr);
 3654   %}
 3655 
 3656   enc_class aarch64_enc_b(label lbl) %{
 3657     C2_MacroAssembler _masm(&cbuf);
 3658     Label *L = $lbl$$label;
 3659     __ b(*L);
 3660   %}
 3661 
 3662   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3663     C2_MacroAssembler _masm(&cbuf);
 3664     Label *L = $lbl$$label;
 3665     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3666   %}
 3667 
 3668   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3669     C2_MacroAssembler _masm(&cbuf);
 3670     Label *L = $lbl$$label;
 3671     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3672   %}
 3673 
 3674   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3675   %{
 3676      Register sub_reg = as_Register($sub$$reg);
 3677      Register super_reg = as_Register($super$$reg);
 3678      Register temp_reg = as_Register($temp$$reg);
 3679      Register result_reg = as_Register($result$$reg);
 3680 
 3681      Label miss;
 3682      C2_MacroAssembler _masm(&cbuf);
 3683      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3684                                      NULL, &miss,
 3685                                      /*set_cond_codes:*/ true);
 3686      if ($primary) {
 3687        __ mov(result_reg, zr);
 3688      }
 3689      __ bind(miss);
 3690   %}
 3691 
 3692   enc_class aarch64_enc_java_static_call(method meth) %{
 3693     C2_MacroAssembler _masm(&cbuf);
 3694 
 3695     address addr = (address)$meth$$method;
 3696     address call;
 3697     if (!_method) {
 3698       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3699       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3700       if (call == NULL) {
 3701         ciEnv::current()->record_failure("CodeCache is full");
 3702         return;
 3703       }
 3704     } else {
 3705       int method_index = resolved_method_index(cbuf);
 3706       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3707                                                   : static_call_Relocation::spec(method_index);
 3708       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3709       if (call == NULL) {
 3710         ciEnv::current()->record_failure("CodeCache is full");
 3711         return;
 3712       }
 3713       // Emit stub for static call
 3714       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3715       if (stub == NULL) {
 3716         ciEnv::current()->record_failure("CodeCache is full");
 3717         return;
 3718       }
 3719     }
 3720 
 3721     // Only non uncommon_trap calls need to reinitialize ptrue.
 3722     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3723       __ reinitialize_ptrue();
 3724     }
 3725   %}
 3726 
 3727   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3728     C2_MacroAssembler _masm(&cbuf);
 3729     int method_index = resolved_method_index(cbuf);
 3730     address call = __ ic_call((address)$meth$$method, method_index);
 3731     if (call == NULL) {
 3732       ciEnv::current()->record_failure("CodeCache is full");
 3733       return;
 3734     } else if (Compile::current()->max_vector_size() > 0) {
 3735       __ reinitialize_ptrue();
 3736     }
 3737   %}
 3738 
 3739   enc_class aarch64_enc_call_epilog() %{
 3740     C2_MacroAssembler _masm(&cbuf);
 3741     if (VerifyStackAtCalls) {
 3742       // Check that stack depth is unchanged: find majik cookie on stack
 3743       __ call_Unimplemented();
 3744     }
 3745   %}
 3746 
 3747   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3748     C2_MacroAssembler _masm(&cbuf);
 3749 
 3750     // some calls to generated routines (arraycopy code) are scheduled
 3751     // by C2 as runtime calls. if so we can call them using a br (they
 3752     // will be in a reachable segment) otherwise we have to use a blr
 3753     // which loads the absolute address into a register.
 3754     address entry = (address)$meth$$method;
 3755     CodeBlob *cb = CodeCache::find_blob(entry);
 3756     if (cb) {
 3757       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3758       if (call == NULL) {
 3759         ciEnv::current()->record_failure("CodeCache is full");
 3760         return;
 3761       }
 3762     } else {
 3763       Label retaddr;
 3764       __ adr(rscratch2, retaddr);
 3765       __ lea(rscratch1, RuntimeAddress(entry));
 3766       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3767       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3768       __ blr(rscratch1);
 3769       __ bind(retaddr);
 3770       __ add(sp, sp, 2 * wordSize);
 3771     }
 3772     if (Compile::current()->max_vector_size() > 0) {
 3773       __ reinitialize_ptrue();
 3774     }
 3775   %}
 3776 
 3777   enc_class aarch64_enc_rethrow() %{
 3778     C2_MacroAssembler _masm(&cbuf);
 3779     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3780   %}
 3781 
 3782   enc_class aarch64_enc_ret() %{
 3783     C2_MacroAssembler _masm(&cbuf);
 3784 #ifdef ASSERT
 3785     if (Compile::current()->max_vector_size() > 0) {
 3786       __ verify_ptrue();
 3787     }
 3788 #endif
 3789     __ ret(lr);
 3790   %}
 3791 
 3792   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3793     C2_MacroAssembler _masm(&cbuf);
 3794     Register target_reg = as_Register($jump_target$$reg);
 3795     __ br(target_reg);
 3796   %}
 3797 
 3798   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3799     C2_MacroAssembler _masm(&cbuf);
 3800     Register target_reg = as_Register($jump_target$$reg);
 3801     // exception oop should be in r0
 3802     // ret addr has been popped into lr
 3803     // callee expects it in r3
 3804     __ mov(r3, lr);
 3805     __ br(target_reg);
 3806   %}
 3807 
 3808   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3809     C2_MacroAssembler _masm(&cbuf);
 3810     Register oop = as_Register($object$$reg);
 3811     Register box = as_Register($box$$reg);
 3812     Register disp_hdr = as_Register($tmp$$reg);
 3813     Register tmp = as_Register($tmp2$$reg);
 3814     Label cont;
 3815     Label object_has_monitor;
 3816     Label cas_failed;
 3817 
 3818     assert_different_registers(oop, box, tmp, disp_hdr);
 3819 
 3820     // Load markWord from object into displaced_header.
 3821     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3822 
 3823     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3824       __ load_klass(tmp, oop);
 3825       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3826       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3827       __ br(Assembler::NE, cont);
 3828     }
 3829 
 3830     // Check for existing monitor
 3831     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3832 
 3833     // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3834     __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3835 
 3836     // Initialize the box. (Must happen before we update the object mark!)
 3837     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3838 
 3839     // Compare object markWord with an unlocked value (tmp) and if
 3840     // equal exchange the stack address of our box with object markWord.
 3841     // On failure disp_hdr contains the possibly locked markWord.
 3842     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3843                /*release*/ true, /*weak*/ false, disp_hdr);
 3844     __ br(Assembler::EQ, cont);
 3845 
 3846     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3847 
 3848     // If the compare-and-exchange succeeded, then we found an unlocked
 3849     // object, will have now locked it will continue at label cont
 3850 
 3851     __ bind(cas_failed);
 3852     // We did not see an unlocked object so try the fast recursive case.
 3853 
 3854     // Check if the owner is self by comparing the value in the
 3855     // markWord of object (disp_hdr) with the stack pointer.
 3856     __ mov(rscratch1, sp);
 3857     __ sub(disp_hdr, disp_hdr, rscratch1);
 3858     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3859     // If condition is true we are cont and hence we can store 0 as the
 3860     // displaced header in the box, which indicates that it is a recursive lock.
 3861     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3862     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3863 
 3864     __ b(cont);
 3865 
 3866     // Handle existing monitor.
 3867     __ bind(object_has_monitor);
 3868 
 3869     // The object's monitor m is unlocked iff m->owner == NULL,
 3870     // otherwise m->owner may contain a thread or a stack address.
 3871     //
 3872     // Try to CAS m->owner from NULL to current thread.
 3873     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3874     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3875                /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
 3876 
 3877     // Store a non-null value into the box to avoid looking like a re-entrant
 3878     // lock. The fast-path monitor unlock code checks for
 3879     // markWord::monitor_value so use markWord::unused_mark which has the
 3880     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3881     __ mov(tmp, (address)markWord::unused_mark().value());
 3882     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3883 
 3884     __ bind(cont);
 3885     // flag == EQ indicates success
 3886     // flag == NE indicates failure
 3887   %}
 3888 
 3889   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3890     C2_MacroAssembler _masm(&cbuf);
 3891     Register oop = as_Register($object$$reg);
 3892     Register box = as_Register($box$$reg);
 3893     Register disp_hdr = as_Register($tmp$$reg);
 3894     Register tmp = as_Register($tmp2$$reg);
 3895     Label cont;
 3896     Label object_has_monitor;
 3897 
 3898     assert_different_registers(oop, box, tmp, disp_hdr);
 3899 
 3900     // Find the lock address and load the displaced header from the stack.
 3901     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3902 
 3903     // If the displaced header is 0, we have a recursive unlock.
 3904     __ cmp(disp_hdr, zr);
 3905     __ br(Assembler::EQ, cont);
 3906 
 3907     // Handle existing monitor.
 3908     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3909     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3910 
 3911     // Check if it is still a light weight lock, this is is true if we
 3912     // see the stack address of the basicLock in the markWord of the
 3913     // object.
 3914 
 3915     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3916                /*release*/ true, /*weak*/ false, tmp);
 3917     __ b(cont);
 3918 
 3919     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3920 
 3921     // Handle existing monitor.
 3922     __ bind(object_has_monitor);
 3923     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3924     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3925     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3926     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3927     __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
 3928     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
 3929     __ cmp(rscratch1, zr); // Sets flags for result
 3930     __ br(Assembler::NE, cont);
 3931 
 3932     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3933     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3934     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3935     __ cmp(rscratch1, zr); // Sets flags for result
 3936     __ cbnz(rscratch1, cont);
 3937     // need a release store here
 3938     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3939     __ stlr(zr, tmp); // set unowned
 3940 
 3941     __ bind(cont);
 3942     // flag == EQ indicates success
 3943     // flag == NE indicates failure
 3944   %}
 3945 
 3946 %}
 3947 
 3948 //----------FRAME--------------------------------------------------------------
 3949 // Definition of frame structure and management information.
 3950 //
 3951 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3952 //                             |   (to get allocators register number
 3953 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3954 //  r   CALLER     |        |
 3955 //  o     |        +--------+      pad to even-align allocators stack-slot
 3956 //  w     V        |  pad0  |        numbers; owned by CALLER
 3957 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3958 //  h     ^        |   in   |  5
 3959 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3960 //  |     |        |        |  3
 3961 //  |     |        +--------+
 3962 //  V     |        | old out|      Empty on Intel, window on Sparc
 3963 //        |    old |preserve|      Must be even aligned.
 3964 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3965 //        |        |   in   |  3   area for Intel ret address
 3966 //     Owned by    |preserve|      Empty on Sparc.
 3967 //       SELF      +--------+
 3968 //        |        |  pad2  |  2   pad to align old SP
 3969 //        |        +--------+  1
 3970 //        |        | locks  |  0
 3971 //        |        +--------+----> OptoReg::stack0(), even aligned
 3972 //        |        |  pad1  | 11   pad to align new SP
 3973 //        |        +--------+
 3974 //        |        |        | 10
 3975 //        |        | spills |  9   spills
 3976 //        V        |        |  8   (pad0 slot for callee)
 3977 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3978 //        ^        |  out   |  7
 3979 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3980 //     Owned by    +--------+
 3981 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3982 //        |    new |preserve|      Must be even-aligned.
 3983 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3984 //        |        |        |
 3985 //
 3986 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3987 //         known from SELF's arguments and the Java calling convention.
 3988 //         Region 6-7 is determined per call site.
 3989 // Note 2: If the calling convention leaves holes in the incoming argument
 3990 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3991 //         are owned by the CALLEE.  Holes should not be nessecary in the
 3992 //         incoming area, as the Java calling convention is completely under
 3993 //         the control of the AD file.  Doubles can be sorted and packed to
 3994 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 3995 //         varargs C calling conventions.
 3996 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3997 //         even aligned with pad0 as needed.
 3998 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3999 //           (the latter is true on Intel but is it false on AArch64?)
 4000 //         region 6-11 is even aligned; it may be padded out more so that
 4001 //         the region from SP to FP meets the minimum stack alignment.
 4002 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4003 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4004 //         SP meets the minimum alignment.
 4005 
 4006 frame %{
 4007   // These three registers define part of the calling convention
 4008   // between compiled code and the interpreter.
 4009 
 4010   // Inline Cache Register or Method for I2C.
 4011   inline_cache_reg(R12);
 4012 
 4013   // Number of stack slots consumed by locking an object
 4014   sync_stack_slots(2);
 4015 
 4016   // Compiled code's Frame Pointer
 4017   frame_pointer(R31);
 4018 
 4019   // Interpreter stores its frame pointer in a register which is
 4020   // stored to the stack by I2CAdaptors.
 4021   // I2CAdaptors convert from interpreted java to compiled java.
 4022   interpreter_frame_pointer(R29);
 4023 
 4024   // Stack alignment requirement
 4025   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4026 
 4027   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4028   // for calls to C.  Supports the var-args backing area for register parms.
 4029   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4030 
 4031   // The after-PROLOG location of the return address.  Location of
 4032   // return address specifies a type (REG or STACK) and a number
 4033   // representing the register number (i.e. - use a register name) or
 4034   // stack slot.
 4035   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4036   // Otherwise, it is above the locks and verification slot and alignment word
 4037   // TODO this may well be correct but need to check why that - 2 is there
 4038   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4039   // which folds in the space used for monitors
 4040   return_addr(STACK - 2 +
 4041               align_up((Compile::current()->in_preserve_stack_slots() +
 4042                         Compile::current()->fixed_slots()),
 4043                        stack_alignment_in_slots()));
 4044 
 4045   // Location of compiled Java return values.  Same as C for now.
 4046   return_value
 4047   %{
 4048     // TODO do we allow ideal_reg == Op_RegN???
 4049     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4050            "only return normal values");
 4051 
 4052     static const int lo[Op_RegL + 1] = { // enum name
 4053       0,                                 // Op_Node
 4054       0,                                 // Op_Set
 4055       R0_num,                            // Op_RegN
 4056       R0_num,                            // Op_RegI
 4057       R0_num,                            // Op_RegP
 4058       V0_num,                            // Op_RegF
 4059       V0_num,                            // Op_RegD
 4060       R0_num                             // Op_RegL
 4061     };
 4062 
 4063     static const int hi[Op_RegL + 1] = { // enum name
 4064       0,                                 // Op_Node
 4065       0,                                 // Op_Set
 4066       OptoReg::Bad,                      // Op_RegN
 4067       OptoReg::Bad,                      // Op_RegI
 4068       R0_H_num,                          // Op_RegP
 4069       OptoReg::Bad,                      // Op_RegF
 4070       V0_H_num,                          // Op_RegD
 4071       R0_H_num                           // Op_RegL
 4072     };
 4073 
 4074     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4075   %}
 4076 %}
 4077 
 4078 //----------ATTRIBUTES---------------------------------------------------------
 4079 //----------Operand Attributes-------------------------------------------------
 4080 op_attrib op_cost(1);        // Required cost attribute
 4081 
 4082 //----------Instruction Attributes---------------------------------------------
 4083 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4084 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4085 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4086                                 // a non-matching short branch variant
 4087                                 // of some long branch?
 4088 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4089                                 // be a power of 2) specifies the
 4090                                 // alignment that some part of the
 4091                                 // instruction (not necessarily the
 4092                                 // start) requires.  If > 1, a
 4093                                 // compute_padding() function must be
 4094                                 // provided for the instruction
 4095 
 4096 //----------OPERANDS-----------------------------------------------------------
 4097 // Operand definitions must precede instruction definitions for correct parsing
 4098 // in the ADLC because operands constitute user defined types which are used in
 4099 // instruction definitions.
 4100 
 4101 //----------Simple Operands----------------------------------------------------
 4102 
 4103 // Integer operands 32 bit
 4104 // 32 bit immediate
 4105 operand immI()
 4106 %{
 4107   match(ConI);
 4108 
 4109   op_cost(0);
 4110   format %{ %}
 4111   interface(CONST_INTER);
 4112 %}
 4113 
 4114 // 32 bit zero
 4115 operand immI0()
 4116 %{
 4117   predicate(n->get_int() == 0);
 4118   match(ConI);
 4119 
 4120   op_cost(0);
 4121   format %{ %}
 4122   interface(CONST_INTER);
 4123 %}
 4124 
 4125 // 32 bit unit increment
 4126 operand immI_1()
 4127 %{
 4128   predicate(n->get_int() == 1);
 4129   match(ConI);
 4130 
 4131   op_cost(0);
 4132   format %{ %}
 4133   interface(CONST_INTER);
 4134 %}
 4135 
 4136 // 32 bit unit decrement
 4137 operand immI_M1()
 4138 %{
 4139   predicate(n->get_int() == -1);
 4140   match(ConI);
 4141 
 4142   op_cost(0);
 4143   format %{ %}
 4144   interface(CONST_INTER);
 4145 %}
 4146 
 4147 // Shift values for add/sub extension shift
 4148 operand immIExt()
 4149 %{
 4150   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4151   match(ConI);
 4152 
 4153   op_cost(0);
 4154   format %{ %}
 4155   interface(CONST_INTER);
 4156 %}
 4157 
 4158 operand immI_gt_1()
 4159 %{
 4160   predicate(n->get_int() > 1);
 4161   match(ConI);
 4162 
 4163   op_cost(0);
 4164   format %{ %}
 4165   interface(CONST_INTER);
 4166 %}
 4167 
 4168 operand immI_le_4()
 4169 %{
 4170   predicate(n->get_int() <= 4);
 4171   match(ConI);
 4172 
 4173   op_cost(0);
 4174   format %{ %}
 4175   interface(CONST_INTER);
 4176 %}
 4177 
 4178 operand immI_31()
 4179 %{
 4180   predicate(n->get_int() == 31);
 4181   match(ConI);
 4182 
 4183   op_cost(0);
 4184   format %{ %}
 4185   interface(CONST_INTER);
 4186 %}
 4187 
 4188 operand immI_2()
 4189 %{
 4190   predicate(n->get_int() == 2);
 4191   match(ConI);
 4192 
 4193   op_cost(0);
 4194   format %{ %}
 4195   interface(CONST_INTER);
 4196 %}
 4197 
 4198 operand immI_4()
 4199 %{
 4200   predicate(n->get_int() == 4);
 4201   match(ConI);
 4202 
 4203   op_cost(0);
 4204   format %{ %}
 4205   interface(CONST_INTER);
 4206 %}
 4207 
 4208 operand immI_8()
 4209 %{
 4210   predicate(n->get_int() == 8);
 4211   match(ConI);
 4212 
 4213   op_cost(0);
 4214   format %{ %}
 4215   interface(CONST_INTER);
 4216 %}
 4217 
 4218 operand immI_16()
 4219 %{
 4220   predicate(n->get_int() == 16);
 4221   match(ConI);
 4222 
 4223   op_cost(0);
 4224   format %{ %}
 4225   interface(CONST_INTER);
 4226 %}
 4227 
 4228 operand immI_24()
 4229 %{
 4230   predicate(n->get_int() == 24);
 4231   match(ConI);
 4232 
 4233   op_cost(0);
 4234   format %{ %}
 4235   interface(CONST_INTER);
 4236 %}
 4237 
 4238 operand immI_32()
 4239 %{
 4240   predicate(n->get_int() == 32);
 4241   match(ConI);
 4242 
 4243   op_cost(0);
 4244   format %{ %}
 4245   interface(CONST_INTER);
 4246 %}
 4247 
 4248 operand immI_48()
 4249 %{
 4250   predicate(n->get_int() == 48);
 4251   match(ConI);
 4252 
 4253   op_cost(0);
 4254   format %{ %}
 4255   interface(CONST_INTER);
 4256 %}
 4257 
 4258 operand immI_56()
 4259 %{
 4260   predicate(n->get_int() == 56);
 4261   match(ConI);
 4262 
 4263   op_cost(0);
 4264   format %{ %}
 4265   interface(CONST_INTER);
 4266 %}
 4267 
 4268 operand immI_63()
 4269 %{
 4270   predicate(n->get_int() == 63);
 4271   match(ConI);
 4272 
 4273   op_cost(0);
 4274   format %{ %}
 4275   interface(CONST_INTER);
 4276 %}
 4277 
 4278 operand immI_64()
 4279 %{
 4280   predicate(n->get_int() == 64);
 4281   match(ConI);
 4282 
 4283   op_cost(0);
 4284   format %{ %}
 4285   interface(CONST_INTER);
 4286 %}
 4287 
 4288 operand immI_255()
 4289 %{
 4290   predicate(n->get_int() == 255);
 4291   match(ConI);
 4292 
 4293   op_cost(0);
 4294   format %{ %}
 4295   interface(CONST_INTER);
 4296 %}
 4297 
 4298 operand immI_65535()
 4299 %{
 4300   predicate(n->get_int() == 65535);
 4301   match(ConI);
 4302 
 4303   op_cost(0);
 4304   format %{ %}
 4305   interface(CONST_INTER);
 4306 %}
 4307 
 4308 operand immL_255()
 4309 %{
 4310   predicate(n->get_long() == 255L);
 4311   match(ConL);
 4312 
 4313   op_cost(0);
 4314   format %{ %}
 4315   interface(CONST_INTER);
 4316 %}
 4317 
 4318 operand immL_65535()
 4319 %{
 4320   predicate(n->get_long() == 65535L);
 4321   match(ConL);
 4322 
 4323   op_cost(0);
 4324   format %{ %}
 4325   interface(CONST_INTER);
 4326 %}
 4327 
 4328 operand immL_4294967295()
 4329 %{
 4330   predicate(n->get_long() == 4294967295L);
 4331   match(ConL);
 4332 
 4333   op_cost(0);
 4334   format %{ %}
 4335   interface(CONST_INTER);
 4336 %}
 4337 
 4338 operand immL_bitmask()
 4339 %{
 4340   predicate((n->get_long() != 0)
 4341             && ((n->get_long() & 0xc000000000000000l) == 0)
 4342             && is_power_of_2(n->get_long() + 1));
 4343   match(ConL);
 4344 
 4345   op_cost(0);
 4346   format %{ %}
 4347   interface(CONST_INTER);
 4348 %}
 4349 
 4350 operand immI_bitmask()
 4351 %{
 4352   predicate((n->get_int() != 0)
 4353             && ((n->get_int() & 0xc0000000) == 0)
 4354             && is_power_of_2(n->get_int() + 1));
 4355   match(ConI);
 4356 
 4357   op_cost(0);
 4358   format %{ %}
 4359   interface(CONST_INTER);
 4360 %}
 4361 
 4362 operand immL_positive_bitmaskI()
 4363 %{
 4364   predicate((n->get_long() != 0)
 4365             && ((julong)n->get_long() < 0x80000000ULL)
 4366             && is_power_of_2(n->get_long() + 1));
 4367   match(ConL);
 4368 
 4369   op_cost(0);
 4370   format %{ %}
 4371   interface(CONST_INTER);
 4372 %}
 4373 
 4374 // Scale values for scaled offset addressing modes (up to long but not quad)
 4375 operand immIScale()
 4376 %{
 4377   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4378   match(ConI);
 4379 
 4380   op_cost(0);
 4381   format %{ %}
 4382   interface(CONST_INTER);
 4383 %}
 4384 
 4385 // 26 bit signed offset -- for pc-relative branches
 4386 operand immI26()
 4387 %{
 4388   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4389   match(ConI);
 4390 
 4391   op_cost(0);
 4392   format %{ %}
 4393   interface(CONST_INTER);
 4394 %}
 4395 
 4396 // 19 bit signed offset -- for pc-relative loads
 4397 operand immI19()
 4398 %{
 4399   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4400   match(ConI);
 4401 
 4402   op_cost(0);
 4403   format %{ %}
 4404   interface(CONST_INTER);
 4405 %}
 4406 
 4407 // 12 bit unsigned offset -- for base plus immediate loads
 4408 operand immIU12()
 4409 %{
 4410   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4411   match(ConI);
 4412 
 4413   op_cost(0);
 4414   format %{ %}
 4415   interface(CONST_INTER);
 4416 %}
 4417 
 4418 operand immLU12()
 4419 %{
 4420   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4421   match(ConL);
 4422 
 4423   op_cost(0);
 4424   format %{ %}
 4425   interface(CONST_INTER);
 4426 %}
 4427 
 4428 // Offset for scaled or unscaled immediate loads and stores
 4429 operand immIOffset()
 4430 %{
 4431   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4432   match(ConI);
 4433 
 4434   op_cost(0);
 4435   format %{ %}
 4436   interface(CONST_INTER);
 4437 %}
 4438 
 4439 operand immIOffset1()
 4440 %{
 4441   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4442   match(ConI);
 4443 
 4444   op_cost(0);
 4445   format %{ %}
 4446   interface(CONST_INTER);
 4447 %}
 4448 
 4449 operand immIOffset2()
 4450 %{
 4451   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4452   match(ConI);
 4453 
 4454   op_cost(0);
 4455   format %{ %}
 4456   interface(CONST_INTER);
 4457 %}
 4458 
 4459 operand immIOffset4()
 4460 %{
 4461   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4462   match(ConI);
 4463 
 4464   op_cost(0);
 4465   format %{ %}
 4466   interface(CONST_INTER);
 4467 %}
 4468 
 4469 operand immIOffset8()
 4470 %{
 4471   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4472   match(ConI);
 4473 
 4474   op_cost(0);
 4475   format %{ %}
 4476   interface(CONST_INTER);
 4477 %}
 4478 
 4479 operand immIOffset16()
 4480 %{
 4481   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4482   match(ConI);
 4483 
 4484   op_cost(0);
 4485   format %{ %}
 4486   interface(CONST_INTER);
 4487 %}
 4488 
 4489 operand immLoffset()
 4490 %{
 4491   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4492   match(ConL);
 4493 
 4494   op_cost(0);
 4495   format %{ %}
 4496   interface(CONST_INTER);
 4497 %}
 4498 
 4499 operand immLoffset1()
 4500 %{
 4501   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4502   match(ConL);
 4503 
 4504   op_cost(0);
 4505   format %{ %}
 4506   interface(CONST_INTER);
 4507 %}
 4508 
 4509 operand immLoffset2()
 4510 %{
 4511   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4512   match(ConL);
 4513 
 4514   op_cost(0);
 4515   format %{ %}
 4516   interface(CONST_INTER);
 4517 %}
 4518 
 4519 operand immLoffset4()
 4520 %{
 4521   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4522   match(ConL);
 4523 
 4524   op_cost(0);
 4525   format %{ %}
 4526   interface(CONST_INTER);
 4527 %}
 4528 
 4529 operand immLoffset8()
 4530 %{
 4531   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4532   match(ConL);
 4533 
 4534   op_cost(0);
 4535   format %{ %}
 4536   interface(CONST_INTER);
 4537 %}
 4538 
 4539 operand immLoffset16()
 4540 %{
 4541   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4542   match(ConL);
 4543 
 4544   op_cost(0);
 4545   format %{ %}
 4546   interface(CONST_INTER);
 4547 %}
 4548 
 4549 // 8 bit signed value.
 4550 operand immI8()
 4551 %{
 4552   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4553   match(ConI);
 4554 
 4555   op_cost(0);
 4556   format %{ %}
 4557   interface(CONST_INTER);
 4558 %}
 4559 
 4560 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4561 operand immI8_shift8()
 4562 %{
 4563   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4564             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4565   match(ConI);
 4566 
 4567   op_cost(0);
 4568   format %{ %}
 4569   interface(CONST_INTER);
 4570 %}
 4571 
 4572 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4573 operand immL8_shift8()
 4574 %{
 4575   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4576             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4577   match(ConL);
 4578 
 4579   op_cost(0);
 4580   format %{ %}
 4581   interface(CONST_INTER);
 4582 %}
 4583 
 4584 // 32 bit integer valid for add sub immediate
 4585 operand immIAddSub()
 4586 %{
 4587   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4588   match(ConI);
 4589   op_cost(0);
 4590   format %{ %}
 4591   interface(CONST_INTER);
 4592 %}
 4593 
 4594 // 32 bit unsigned integer valid for logical immediate
 4595 // TODO -- check this is right when e.g the mask is 0x80000000
 4596 operand immILog()
 4597 %{
 4598   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4599   match(ConI);
 4600 
 4601   op_cost(0);
 4602   format %{ %}
 4603   interface(CONST_INTER);
 4604 %}
 4605 
 4606 // Integer operands 64 bit
 4607 // 64 bit immediate
 4608 operand immL()
 4609 %{
 4610   match(ConL);
 4611 
 4612   op_cost(0);
 4613   format %{ %}
 4614   interface(CONST_INTER);
 4615 %}
 4616 
 4617 // 64 bit zero
 4618 operand immL0()
 4619 %{
 4620   predicate(n->get_long() == 0);
 4621   match(ConL);
 4622 
 4623   op_cost(0);
 4624   format %{ %}
 4625   interface(CONST_INTER);
 4626 %}
 4627 
 4628 // 64 bit unit increment
 4629 operand immL_1()
 4630 %{
 4631   predicate(n->get_long() == 1);
 4632   match(ConL);
 4633 
 4634   op_cost(0);
 4635   format %{ %}
 4636   interface(CONST_INTER);
 4637 %}
 4638 
 4639 // 64 bit unit decrement
 4640 operand immL_M1()
 4641 %{
 4642   predicate(n->get_long() == -1);
 4643   match(ConL);
 4644 
 4645   op_cost(0);
 4646   format %{ %}
 4647   interface(CONST_INTER);
 4648 %}
 4649 
 4650 // 32 bit offset of pc in thread anchor
 4651 
 4652 operand immL_pc_off()
 4653 %{
 4654   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4655                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4656   match(ConL);
 4657 
 4658   op_cost(0);
 4659   format %{ %}
 4660   interface(CONST_INTER);
 4661 %}
 4662 
 4663 // 64 bit integer valid for add sub immediate
 4664 operand immLAddSub()
 4665 %{
 4666   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4667   match(ConL);
 4668   op_cost(0);
 4669   format %{ %}
 4670   interface(CONST_INTER);
 4671 %}
 4672 
 4673 // 64 bit integer valid for logical immediate
 4674 operand immLLog()
 4675 %{
 4676   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4677   match(ConL);
 4678   op_cost(0);
 4679   format %{ %}
 4680   interface(CONST_INTER);
 4681 %}
 4682 
 4683 // Long Immediate: low 32-bit mask
 4684 operand immL_32bits()
 4685 %{
 4686   predicate(n->get_long() == 0xFFFFFFFFL);
 4687   match(ConL);
 4688   op_cost(0);
 4689   format %{ %}
 4690   interface(CONST_INTER);
 4691 %}
 4692 
 4693 // Pointer operands
 4694 // Pointer Immediate
 4695 operand immP()
 4696 %{
 4697   match(ConP);
 4698 
 4699   op_cost(0);
 4700   format %{ %}
 4701   interface(CONST_INTER);
 4702 %}
 4703 
 4704 // NULL Pointer Immediate
 4705 operand immP0()
 4706 %{
 4707   predicate(n->get_ptr() == 0);
 4708   match(ConP);
 4709 
 4710   op_cost(0);
 4711   format %{ %}
 4712   interface(CONST_INTER);
 4713 %}
 4714 
 4715 // Pointer Immediate One
 4716 // this is used in object initialization (initial object header)
 4717 operand immP_1()
 4718 %{
 4719   predicate(n->get_ptr() == 1);
 4720   match(ConP);
 4721 
 4722   op_cost(0);
 4723   format %{ %}
 4724   interface(CONST_INTER);
 4725 %}
 4726 
 4727 // Card Table Byte Map Base
 4728 operand immByteMapBase()
 4729 %{
 4730   // Get base of card map
 4731   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4732             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4733   match(ConP);
 4734 
 4735   op_cost(0);
 4736   format %{ %}
 4737   interface(CONST_INTER);
 4738 %}
 4739 
 4740 // Pointer Immediate Minus One
 4741 // this is used when we want to write the current PC to the thread anchor
 4742 operand immP_M1()
 4743 %{
 4744   predicate(n->get_ptr() == -1);
 4745   match(ConP);
 4746 
 4747   op_cost(0);
 4748   format %{ %}
 4749   interface(CONST_INTER);
 4750 %}
 4751 
 4752 // Pointer Immediate Minus Two
 4753 // this is used when we want to write the current PC to the thread anchor
 4754 operand immP_M2()
 4755 %{
 4756   predicate(n->get_ptr() == -2);
 4757   match(ConP);
 4758 
 4759   op_cost(0);
 4760   format %{ %}
 4761   interface(CONST_INTER);
 4762 %}
 4763 
 4764 // Float and Double operands
 4765 // Double Immediate
 4766 operand immD()
 4767 %{
 4768   match(ConD);
 4769   op_cost(0);
 4770   format %{ %}
 4771   interface(CONST_INTER);
 4772 %}
 4773 
 4774 // Double Immediate: +0.0d
 4775 operand immD0()
 4776 %{
 4777   predicate(jlong_cast(n->getd()) == 0);
 4778   match(ConD);
 4779 
 4780   op_cost(0);
 4781   format %{ %}
 4782   interface(CONST_INTER);
 4783 %}
 4784 
 4785 // constant 'double +0.0'.
 4786 operand immDPacked()
 4787 %{
 4788   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4789   match(ConD);
 4790   op_cost(0);
 4791   format %{ %}
 4792   interface(CONST_INTER);
 4793 %}
 4794 
 4795 // Float Immediate
 4796 operand immF()
 4797 %{
 4798   match(ConF);
 4799   op_cost(0);
 4800   format %{ %}
 4801   interface(CONST_INTER);
 4802 %}
 4803 
 4804 // Float Immediate: +0.0f.
 4805 operand immF0()
 4806 %{
 4807   predicate(jint_cast(n->getf()) == 0);
 4808   match(ConF);
 4809 
 4810   op_cost(0);
 4811   format %{ %}
 4812   interface(CONST_INTER);
 4813 %}
 4814 
 4815 //
 4816 operand immFPacked()
 4817 %{
 4818   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4819   match(ConF);
 4820   op_cost(0);
 4821   format %{ %}
 4822   interface(CONST_INTER);
 4823 %}
 4824 
 4825 // Narrow pointer operands
 4826 // Narrow Pointer Immediate
 4827 operand immN()
 4828 %{
 4829   match(ConN);
 4830 
 4831   op_cost(0);
 4832   format %{ %}
 4833   interface(CONST_INTER);
 4834 %}
 4835 
 4836 // Narrow NULL Pointer Immediate
 4837 operand immN0()
 4838 %{
 4839   predicate(n->get_narrowcon() == 0);
 4840   match(ConN);
 4841 
 4842   op_cost(0);
 4843   format %{ %}
 4844   interface(CONST_INTER);
 4845 %}
 4846 
 4847 operand immNKlass()
 4848 %{
 4849   match(ConNKlass);
 4850 
 4851   op_cost(0);
 4852   format %{ %}
 4853   interface(CONST_INTER);
 4854 %}
 4855 
 4856 // Integer 32 bit Register Operands
 4857 // Integer 32 bitRegister (excludes SP)
 4858 operand iRegI()
 4859 %{
 4860   constraint(ALLOC_IN_RC(any_reg32));
 4861   match(RegI);
 4862   match(iRegINoSp);
 4863   op_cost(0);
 4864   format %{ %}
 4865   interface(REG_INTER);
 4866 %}
 4867 
 4868 // Integer 32 bit Register not Special
 4869 operand iRegINoSp()
 4870 %{
 4871   constraint(ALLOC_IN_RC(no_special_reg32));
 4872   match(RegI);
 4873   op_cost(0);
 4874   format %{ %}
 4875   interface(REG_INTER);
 4876 %}
 4877 
 4878 // Integer 64 bit Register Operands
 4879 // Integer 64 bit Register (includes SP)
 4880 operand iRegL()
 4881 %{
 4882   constraint(ALLOC_IN_RC(any_reg));
 4883   match(RegL);
 4884   match(iRegLNoSp);
 4885   op_cost(0);
 4886   format %{ %}
 4887   interface(REG_INTER);
 4888 %}
 4889 
 4890 // Integer 64 bit Register not Special
 4891 operand iRegLNoSp()
 4892 %{
 4893   constraint(ALLOC_IN_RC(no_special_reg));
 4894   match(RegL);
 4895   match(iRegL_R0);
 4896   format %{ %}
 4897   interface(REG_INTER);
 4898 %}
 4899 
 4900 // Pointer Register Operands
 4901 // Pointer Register
 4902 operand iRegP()
 4903 %{
 4904   constraint(ALLOC_IN_RC(ptr_reg));
 4905   match(RegP);
 4906   match(iRegPNoSp);
 4907   match(iRegP_R0);
 4908   //match(iRegP_R2);
 4909   //match(iRegP_R4);
 4910   //match(iRegP_R5);
 4911   match(thread_RegP);
 4912   op_cost(0);
 4913   format %{ %}
 4914   interface(REG_INTER);
 4915 %}
 4916 
 4917 // Pointer 64 bit Register not Special
 4918 operand iRegPNoSp()
 4919 %{
 4920   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4921   match(RegP);
 4922   // match(iRegP);
 4923   // match(iRegP_R0);
 4924   // match(iRegP_R2);
 4925   // match(iRegP_R4);
 4926   // match(iRegP_R5);
 4927   // match(thread_RegP);
 4928   op_cost(0);
 4929   format %{ %}
 4930   interface(REG_INTER);
 4931 %}
 4932 
 4933 // Pointer 64 bit Register R0 only
 4934 operand iRegP_R0()
 4935 %{
 4936   constraint(ALLOC_IN_RC(r0_reg));
 4937   match(RegP);
 4938   // match(iRegP);
 4939   match(iRegPNoSp);
 4940   op_cost(0);
 4941   format %{ %}
 4942   interface(REG_INTER);
 4943 %}
 4944 
 4945 // Pointer 64 bit Register R1 only
 4946 operand iRegP_R1()
 4947 %{
 4948   constraint(ALLOC_IN_RC(r1_reg));
 4949   match(RegP);
 4950   // match(iRegP);
 4951   match(iRegPNoSp);
 4952   op_cost(0);
 4953   format %{ %}
 4954   interface(REG_INTER);
 4955 %}
 4956 
 4957 // Pointer 64 bit Register R2 only
 4958 operand iRegP_R2()
 4959 %{
 4960   constraint(ALLOC_IN_RC(r2_reg));
 4961   match(RegP);
 4962   // match(iRegP);
 4963   match(iRegPNoSp);
 4964   op_cost(0);
 4965   format %{ %}
 4966   interface(REG_INTER);
 4967 %}
 4968 
 4969 // Pointer 64 bit Register R3 only
 4970 operand iRegP_R3()
 4971 %{
 4972   constraint(ALLOC_IN_RC(r3_reg));
 4973   match(RegP);
 4974   // match(iRegP);
 4975   match(iRegPNoSp);
 4976   op_cost(0);
 4977   format %{ %}
 4978   interface(REG_INTER);
 4979 %}
 4980 
 4981 // Pointer 64 bit Register R4 only
 4982 operand iRegP_R4()
 4983 %{
 4984   constraint(ALLOC_IN_RC(r4_reg));
 4985   match(RegP);
 4986   // match(iRegP);
 4987   match(iRegPNoSp);
 4988   op_cost(0);
 4989   format %{ %}
 4990   interface(REG_INTER);
 4991 %}
 4992 
 4993 // Pointer 64 bit Register R5 only
 4994 operand iRegP_R5()
 4995 %{
 4996   constraint(ALLOC_IN_RC(r5_reg));
 4997   match(RegP);
 4998   // match(iRegP);
 4999   match(iRegPNoSp);
 5000   op_cost(0);
 5001   format %{ %}
 5002   interface(REG_INTER);
 5003 %}
 5004 
 5005 // Pointer 64 bit Register R10 only
 5006 operand iRegP_R10()
 5007 %{
 5008   constraint(ALLOC_IN_RC(r10_reg));
 5009   match(RegP);
 5010   // match(iRegP);
 5011   match(iRegPNoSp);
 5012   op_cost(0);
 5013   format %{ %}
 5014   interface(REG_INTER);
 5015 %}
 5016 
 5017 // Long 64 bit Register R0 only
 5018 operand iRegL_R0()
 5019 %{
 5020   constraint(ALLOC_IN_RC(r0_reg));
 5021   match(RegL);
 5022   match(iRegLNoSp);
 5023   op_cost(0);
 5024   format %{ %}
 5025   interface(REG_INTER);
 5026 %}
 5027 
 5028 // Long 64 bit Register R2 only
 5029 operand iRegL_R2()
 5030 %{
 5031   constraint(ALLOC_IN_RC(r2_reg));
 5032   match(RegL);
 5033   match(iRegLNoSp);
 5034   op_cost(0);
 5035   format %{ %}
 5036   interface(REG_INTER);
 5037 %}
 5038 
 5039 // Long 64 bit Register R3 only
 5040 operand iRegL_R3()
 5041 %{
 5042   constraint(ALLOC_IN_RC(r3_reg));
 5043   match(RegL);
 5044   match(iRegLNoSp);
 5045   op_cost(0);
 5046   format %{ %}
 5047   interface(REG_INTER);
 5048 %}
 5049 
 5050 // Long 64 bit Register R11 only
 5051 operand iRegL_R11()
 5052 %{
 5053   constraint(ALLOC_IN_RC(r11_reg));
 5054   match(RegL);
 5055   match(iRegLNoSp);
 5056   op_cost(0);
 5057   format %{ %}
 5058   interface(REG_INTER);
 5059 %}
 5060 
 5061 // Pointer 64 bit Register FP only
 5062 operand iRegP_FP()
 5063 %{
 5064   constraint(ALLOC_IN_RC(fp_reg));
 5065   match(RegP);
 5066   // match(iRegP);
 5067   op_cost(0);
 5068   format %{ %}
 5069   interface(REG_INTER);
 5070 %}
 5071 
 5072 // Register R0 only
 5073 operand iRegI_R0()
 5074 %{
 5075   constraint(ALLOC_IN_RC(int_r0_reg));
 5076   match(RegI);
 5077   match(iRegINoSp);
 5078   op_cost(0);
 5079   format %{ %}
 5080   interface(REG_INTER);
 5081 %}
 5082 
 5083 // Register R2 only
 5084 operand iRegI_R2()
 5085 %{
 5086   constraint(ALLOC_IN_RC(int_r2_reg));
 5087   match(RegI);
 5088   match(iRegINoSp);
 5089   op_cost(0);
 5090   format %{ %}
 5091   interface(REG_INTER);
 5092 %}
 5093 
 5094 // Register R3 only
 5095 operand iRegI_R3()
 5096 %{
 5097   constraint(ALLOC_IN_RC(int_r3_reg));
 5098   match(RegI);
 5099   match(iRegINoSp);
 5100   op_cost(0);
 5101   format %{ %}
 5102   interface(REG_INTER);
 5103 %}
 5104 
 5105 
 5106 // Register R4 only
 5107 operand iRegI_R4()
 5108 %{
 5109   constraint(ALLOC_IN_RC(int_r4_reg));
 5110   match(RegI);
 5111   match(iRegINoSp);
 5112   op_cost(0);
 5113   format %{ %}
 5114   interface(REG_INTER);
 5115 %}
 5116 
 5117 
 5118 // Pointer Register Operands
 5119 // Narrow Pointer Register
 5120 operand iRegN()
 5121 %{
 5122   constraint(ALLOC_IN_RC(any_reg32));
 5123   match(RegN);
 5124   match(iRegNNoSp);
 5125   op_cost(0);
 5126   format %{ %}
 5127   interface(REG_INTER);
 5128 %}
 5129 
 5130 operand iRegN_R0()
 5131 %{
 5132   constraint(ALLOC_IN_RC(r0_reg));
 5133   match(iRegN);
 5134   op_cost(0);
 5135   format %{ %}
 5136   interface(REG_INTER);
 5137 %}
 5138 
 5139 operand iRegN_R2()
 5140 %{
 5141   constraint(ALLOC_IN_RC(r2_reg));
 5142   match(iRegN);
 5143   op_cost(0);
 5144   format %{ %}
 5145   interface(REG_INTER);
 5146 %}
 5147 
 5148 operand iRegN_R3()
 5149 %{
 5150   constraint(ALLOC_IN_RC(r3_reg));
 5151   match(iRegN);
 5152   op_cost(0);
 5153   format %{ %}
 5154   interface(REG_INTER);
 5155 %}
 5156 
 5157 // Integer 64 bit Register not Special
 5158 operand iRegNNoSp()
 5159 %{
 5160   constraint(ALLOC_IN_RC(no_special_reg32));
 5161   match(RegN);
 5162   op_cost(0);
 5163   format %{ %}
 5164   interface(REG_INTER);
 5165 %}
 5166 
 5167 // heap base register -- used for encoding immN0
 5168 
 5169 operand iRegIHeapbase()
 5170 %{
 5171   constraint(ALLOC_IN_RC(heapbase_reg));
 5172   match(RegI);
 5173   op_cost(0);
 5174   format %{ %}
 5175   interface(REG_INTER);
 5176 %}
 5177 
 5178 // Float Register
 5179 // Float register operands
 5180 operand vRegF()
 5181 %{
 5182   constraint(ALLOC_IN_RC(float_reg));
 5183   match(RegF);
 5184 
 5185   op_cost(0);
 5186   format %{ %}
 5187   interface(REG_INTER);
 5188 %}
 5189 
 5190 // Double Register
 5191 // Double register operands
 5192 operand vRegD()
 5193 %{
 5194   constraint(ALLOC_IN_RC(double_reg));
 5195   match(RegD);
 5196 
 5197   op_cost(0);
 5198   format %{ %}
 5199   interface(REG_INTER);
 5200 %}
 5201 
 5202 // Generic vector class. This will be used for
 5203 // all vector operands, including NEON and SVE,
 5204 // but currently only used for SVE VecA.
 5205 operand vReg()
 5206 %{
 5207   constraint(ALLOC_IN_RC(vectora_reg));
 5208   match(VecA);
 5209   op_cost(0);
 5210   format %{ %}
 5211   interface(REG_INTER);
 5212 %}
 5213 
 5214 operand vecD()
 5215 %{
 5216   constraint(ALLOC_IN_RC(vectord_reg));
 5217   match(VecD);
 5218 
 5219   op_cost(0);
 5220   format %{ %}
 5221   interface(REG_INTER);
 5222 %}
 5223 
 5224 operand vecX()
 5225 %{
 5226   constraint(ALLOC_IN_RC(vectorx_reg));
 5227   match(VecX);
 5228 
 5229   op_cost(0);
 5230   format %{ %}
 5231   interface(REG_INTER);
 5232 %}
 5233 
 5234 operand vRegD_V0()
 5235 %{
 5236   constraint(ALLOC_IN_RC(v0_reg));
 5237   match(RegD);
 5238   op_cost(0);
 5239   format %{ %}
 5240   interface(REG_INTER);
 5241 %}
 5242 
 5243 operand vRegD_V1()
 5244 %{
 5245   constraint(ALLOC_IN_RC(v1_reg));
 5246   match(RegD);
 5247   op_cost(0);
 5248   format %{ %}
 5249   interface(REG_INTER);
 5250 %}
 5251 
 5252 operand vRegD_V2()
 5253 %{
 5254   constraint(ALLOC_IN_RC(v2_reg));
 5255   match(RegD);
 5256   op_cost(0);
 5257   format %{ %}
 5258   interface(REG_INTER);
 5259 %}
 5260 
 5261 operand vRegD_V3()
 5262 %{
 5263   constraint(ALLOC_IN_RC(v3_reg));
 5264   match(RegD);
 5265   op_cost(0);
 5266   format %{ %}
 5267   interface(REG_INTER);
 5268 %}
 5269 
 5270 operand vRegD_V4()
 5271 %{
 5272   constraint(ALLOC_IN_RC(v4_reg));
 5273   match(RegD);
 5274   op_cost(0);
 5275   format %{ %}
 5276   interface(REG_INTER);
 5277 %}
 5278 
 5279 operand vRegD_V5()
 5280 %{
 5281   constraint(ALLOC_IN_RC(v5_reg));
 5282   match(RegD);
 5283   op_cost(0);
 5284   format %{ %}
 5285   interface(REG_INTER);
 5286 %}
 5287 
 5288 operand vRegD_V6()
 5289 %{
 5290   constraint(ALLOC_IN_RC(v6_reg));
 5291   match(RegD);
 5292   op_cost(0);
 5293   format %{ %}
 5294   interface(REG_INTER);
 5295 %}
 5296 
 5297 operand vRegD_V7()
 5298 %{
 5299   constraint(ALLOC_IN_RC(v7_reg));
 5300   match(RegD);
 5301   op_cost(0);
 5302   format %{ %}
 5303   interface(REG_INTER);
 5304 %}
 5305 
 5306 operand vRegD_V8()
 5307 %{
 5308   constraint(ALLOC_IN_RC(v8_reg));
 5309   match(RegD);
 5310   op_cost(0);
 5311   format %{ %}
 5312   interface(REG_INTER);
 5313 %}
 5314 
 5315 operand vRegD_V9()
 5316 %{
 5317   constraint(ALLOC_IN_RC(v9_reg));
 5318   match(RegD);
 5319   op_cost(0);
 5320   format %{ %}
 5321   interface(REG_INTER);
 5322 %}
 5323 
 5324 operand vRegD_V10()
 5325 %{
 5326   constraint(ALLOC_IN_RC(v10_reg));
 5327   match(RegD);
 5328   op_cost(0);
 5329   format %{ %}
 5330   interface(REG_INTER);
 5331 %}
 5332 
 5333 operand vRegD_V11()
 5334 %{
 5335   constraint(ALLOC_IN_RC(v11_reg));
 5336   match(RegD);
 5337   op_cost(0);
 5338   format %{ %}
 5339   interface(REG_INTER);
 5340 %}
 5341 
 5342 operand vRegD_V12()
 5343 %{
 5344   constraint(ALLOC_IN_RC(v12_reg));
 5345   match(RegD);
 5346   op_cost(0);
 5347   format %{ %}
 5348   interface(REG_INTER);
 5349 %}
 5350 
 5351 operand vRegD_V13()
 5352 %{
 5353   constraint(ALLOC_IN_RC(v13_reg));
 5354   match(RegD);
 5355   op_cost(0);
 5356   format %{ %}
 5357   interface(REG_INTER);
 5358 %}
 5359 
 5360 operand vRegD_V14()
 5361 %{
 5362   constraint(ALLOC_IN_RC(v14_reg));
 5363   match(RegD);
 5364   op_cost(0);
 5365   format %{ %}
 5366   interface(REG_INTER);
 5367 %}
 5368 
 5369 operand vRegD_V15()
 5370 %{
 5371   constraint(ALLOC_IN_RC(v15_reg));
 5372   match(RegD);
 5373   op_cost(0);
 5374   format %{ %}
 5375   interface(REG_INTER);
 5376 %}
 5377 
 5378 operand vRegD_V16()
 5379 %{
 5380   constraint(ALLOC_IN_RC(v16_reg));
 5381   match(RegD);
 5382   op_cost(0);
 5383   format %{ %}
 5384   interface(REG_INTER);
 5385 %}
 5386 
 5387 operand vRegD_V17()
 5388 %{
 5389   constraint(ALLOC_IN_RC(v17_reg));
 5390   match(RegD);
 5391   op_cost(0);
 5392   format %{ %}
 5393   interface(REG_INTER);
 5394 %}
 5395 
 5396 operand vRegD_V18()
 5397 %{
 5398   constraint(ALLOC_IN_RC(v18_reg));
 5399   match(RegD);
 5400   op_cost(0);
 5401   format %{ %}
 5402   interface(REG_INTER);
 5403 %}
 5404 
 5405 operand vRegD_V19()
 5406 %{
 5407   constraint(ALLOC_IN_RC(v19_reg));
 5408   match(RegD);
 5409   op_cost(0);
 5410   format %{ %}
 5411   interface(REG_INTER);
 5412 %}
 5413 
 5414 operand vRegD_V20()
 5415 %{
 5416   constraint(ALLOC_IN_RC(v20_reg));
 5417   match(RegD);
 5418   op_cost(0);
 5419   format %{ %}
 5420   interface(REG_INTER);
 5421 %}
 5422 
 5423 operand vRegD_V21()
 5424 %{
 5425   constraint(ALLOC_IN_RC(v21_reg));
 5426   match(RegD);
 5427   op_cost(0);
 5428   format %{ %}
 5429   interface(REG_INTER);
 5430 %}
 5431 
 5432 operand vRegD_V22()
 5433 %{
 5434   constraint(ALLOC_IN_RC(v22_reg));
 5435   match(RegD);
 5436   op_cost(0);
 5437   format %{ %}
 5438   interface(REG_INTER);
 5439 %}
 5440 
 5441 operand vRegD_V23()
 5442 %{
 5443   constraint(ALLOC_IN_RC(v23_reg));
 5444   match(RegD);
 5445   op_cost(0);
 5446   format %{ %}
 5447   interface(REG_INTER);
 5448 %}
 5449 
 5450 operand vRegD_V24()
 5451 %{
 5452   constraint(ALLOC_IN_RC(v24_reg));
 5453   match(RegD);
 5454   op_cost(0);
 5455   format %{ %}
 5456   interface(REG_INTER);
 5457 %}
 5458 
 5459 operand vRegD_V25()
 5460 %{
 5461   constraint(ALLOC_IN_RC(v25_reg));
 5462   match(RegD);
 5463   op_cost(0);
 5464   format %{ %}
 5465   interface(REG_INTER);
 5466 %}
 5467 
 5468 operand vRegD_V26()
 5469 %{
 5470   constraint(ALLOC_IN_RC(v26_reg));
 5471   match(RegD);
 5472   op_cost(0);
 5473   format %{ %}
 5474   interface(REG_INTER);
 5475 %}
 5476 
 5477 operand vRegD_V27()
 5478 %{
 5479   constraint(ALLOC_IN_RC(v27_reg));
 5480   match(RegD);
 5481   op_cost(0);
 5482   format %{ %}
 5483   interface(REG_INTER);
 5484 %}
 5485 
 5486 operand vRegD_V28()
 5487 %{
 5488   constraint(ALLOC_IN_RC(v28_reg));
 5489   match(RegD);
 5490   op_cost(0);
 5491   format %{ %}
 5492   interface(REG_INTER);
 5493 %}
 5494 
 5495 operand vRegD_V29()
 5496 %{
 5497   constraint(ALLOC_IN_RC(v29_reg));
 5498   match(RegD);
 5499   op_cost(0);
 5500   format %{ %}
 5501   interface(REG_INTER);
 5502 %}
 5503 
 5504 operand vRegD_V30()
 5505 %{
 5506   constraint(ALLOC_IN_RC(v30_reg));
 5507   match(RegD);
 5508   op_cost(0);
 5509   format %{ %}
 5510   interface(REG_INTER);
 5511 %}
 5512 
 5513 operand vRegD_V31()
 5514 %{
 5515   constraint(ALLOC_IN_RC(v31_reg));
 5516   match(RegD);
 5517   op_cost(0);
 5518   format %{ %}
 5519   interface(REG_INTER);
 5520 %}
 5521 
 5522 operand pReg()
 5523 %{
 5524   constraint(ALLOC_IN_RC(pr_reg));
 5525   match(RegVectMask);
 5526   op_cost(0);
 5527   format %{ %}
 5528   interface(REG_INTER);
 5529 %}
 5530 
 5531 operand pRegGov()
 5532 %{
 5533   constraint(ALLOC_IN_RC(gov_pr));
 5534   match(RegVectMask);
 5535   op_cost(0);
 5536   format %{ %}
 5537   interface(REG_INTER);
 5538 %}
 5539 
 5540 // Flags register, used as output of signed compare instructions
 5541 
 5542 // note that on AArch64 we also use this register as the output for
 5543 // for floating point compare instructions (CmpF CmpD). this ensures
 5544 // that ordered inequality tests use GT, GE, LT or LE none of which
 5545 // pass through cases where the result is unordered i.e. one or both
 5546 // inputs to the compare is a NaN. this means that the ideal code can
 5547 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5548 // (where the comparison should always fail). EQ and NE tests are
 5549 // always generated in ideal code so that unordered folds into the NE
 5550 // case, matching the behaviour of AArch64 NE.
 5551 //
 5552 // This differs from x86 where the outputs of FP compares use a
 5553 // special FP flags registers and where compares based on this
 5554 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5555 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5556 // to explicitly handle the unordered case in branches. x86 also has
 5557 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5558 
 5559 operand rFlagsReg()
 5560 %{
 5561   constraint(ALLOC_IN_RC(int_flags));
 5562   match(RegFlags);
 5563 
 5564   op_cost(0);
 5565   format %{ "RFLAGS" %}
 5566   interface(REG_INTER);
 5567 %}
 5568 
 5569 // Flags register, used as output of unsigned compare instructions
 5570 operand rFlagsRegU()
 5571 %{
 5572   constraint(ALLOC_IN_RC(int_flags));
 5573   match(RegFlags);
 5574 
 5575   op_cost(0);
 5576   format %{ "RFLAGSU" %}
 5577   interface(REG_INTER);
 5578 %}
 5579 
 5580 // Special Registers
 5581 
 5582 // Method Register
 5583 operand inline_cache_RegP(iRegP reg)
 5584 %{
 5585   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5586   match(reg);
 5587   match(iRegPNoSp);
 5588   op_cost(0);
 5589   format %{ %}
 5590   interface(REG_INTER);
 5591 %}
 5592 
 5593 // Thread Register
 5594 operand thread_RegP(iRegP reg)
 5595 %{
 5596   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5597   match(reg);
 5598   op_cost(0);
 5599   format %{ %}
 5600   interface(REG_INTER);
 5601 %}
 5602 
 5603 operand lr_RegP(iRegP reg)
 5604 %{
 5605   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5606   match(reg);
 5607   op_cost(0);
 5608   format %{ %}
 5609   interface(REG_INTER);
 5610 %}
 5611 
 5612 //----------Memory Operands----------------------------------------------------
 5613 
 5614 operand indirect(iRegP reg)
 5615 %{
 5616   constraint(ALLOC_IN_RC(ptr_reg));
 5617   match(reg);
 5618   op_cost(0);
 5619   format %{ "[$reg]" %}
 5620   interface(MEMORY_INTER) %{
 5621     base($reg);
 5622     index(0xffffffff);
 5623     scale(0x0);
 5624     disp(0x0);
 5625   %}
 5626 %}
 5627 
 5628 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5629 %{
 5630   constraint(ALLOC_IN_RC(ptr_reg));
 5631   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5632   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5633   op_cost(0);
 5634   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5635   interface(MEMORY_INTER) %{
 5636     base($reg);
 5637     index($ireg);
 5638     scale($scale);
 5639     disp(0x0);
 5640   %}
 5641 %}
 5642 
 5643 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5644 %{
 5645   constraint(ALLOC_IN_RC(ptr_reg));
 5646   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5647   match(AddP reg (LShiftL lreg scale));
 5648   op_cost(0);
 5649   format %{ "$reg, $lreg lsl($scale)" %}
 5650   interface(MEMORY_INTER) %{
 5651     base($reg);
 5652     index($lreg);
 5653     scale($scale);
 5654     disp(0x0);
 5655   %}
 5656 %}
 5657 
 5658 operand indIndexI2L(iRegP reg, iRegI ireg)
 5659 %{
 5660   constraint(ALLOC_IN_RC(ptr_reg));
 5661   match(AddP reg (ConvI2L ireg));
 5662   op_cost(0);
 5663   format %{ "$reg, $ireg, 0, I2L" %}
 5664   interface(MEMORY_INTER) %{
 5665     base($reg);
 5666     index($ireg);
 5667     scale(0x0);
 5668     disp(0x0);
 5669   %}
 5670 %}
 5671 
 5672 operand indIndex(iRegP reg, iRegL lreg)
 5673 %{
 5674   constraint(ALLOC_IN_RC(ptr_reg));
 5675   match(AddP reg lreg);
 5676   op_cost(0);
 5677   format %{ "$reg, $lreg" %}
 5678   interface(MEMORY_INTER) %{
 5679     base($reg);
 5680     index($lreg);
 5681     scale(0x0);
 5682     disp(0x0);
 5683   %}
 5684 %}
 5685 
 5686 operand indOffI(iRegP reg, immIOffset off)
 5687 %{
 5688   constraint(ALLOC_IN_RC(ptr_reg));
 5689   match(AddP reg off);
 5690   op_cost(0);
 5691   format %{ "[$reg, $off]" %}
 5692   interface(MEMORY_INTER) %{
 5693     base($reg);
 5694     index(0xffffffff);
 5695     scale(0x0);
 5696     disp($off);
 5697   %}
 5698 %}
 5699 
 5700 operand indOffI1(iRegP reg, immIOffset1 off)
 5701 %{
 5702   constraint(ALLOC_IN_RC(ptr_reg));
 5703   match(AddP reg off);
 5704   op_cost(0);
 5705   format %{ "[$reg, $off]" %}
 5706   interface(MEMORY_INTER) %{
 5707     base($reg);
 5708     index(0xffffffff);
 5709     scale(0x0);
 5710     disp($off);
 5711   %}
 5712 %}
 5713 
 5714 operand indOffI2(iRegP reg, immIOffset2 off)
 5715 %{
 5716   constraint(ALLOC_IN_RC(ptr_reg));
 5717   match(AddP reg off);
 5718   op_cost(0);
 5719   format %{ "[$reg, $off]" %}
 5720   interface(MEMORY_INTER) %{
 5721     base($reg);
 5722     index(0xffffffff);
 5723     scale(0x0);
 5724     disp($off);
 5725   %}
 5726 %}
 5727 
 5728 operand indOffI4(iRegP reg, immIOffset4 off)
 5729 %{
 5730   constraint(ALLOC_IN_RC(ptr_reg));
 5731   match(AddP reg off);
 5732   op_cost(0);
 5733   format %{ "[$reg, $off]" %}
 5734   interface(MEMORY_INTER) %{
 5735     base($reg);
 5736     index(0xffffffff);
 5737     scale(0x0);
 5738     disp($off);
 5739   %}
 5740 %}
 5741 
 5742 operand indOffI8(iRegP reg, immIOffset8 off)
 5743 %{
 5744   constraint(ALLOC_IN_RC(ptr_reg));
 5745   match(AddP reg off);
 5746   op_cost(0);
 5747   format %{ "[$reg, $off]" %}
 5748   interface(MEMORY_INTER) %{
 5749     base($reg);
 5750     index(0xffffffff);
 5751     scale(0x0);
 5752     disp($off);
 5753   %}
 5754 %}
 5755 
 5756 operand indOffI16(iRegP reg, immIOffset16 off)
 5757 %{
 5758   constraint(ALLOC_IN_RC(ptr_reg));
 5759   match(AddP reg off);
 5760   op_cost(0);
 5761   format %{ "[$reg, $off]" %}
 5762   interface(MEMORY_INTER) %{
 5763     base($reg);
 5764     index(0xffffffff);
 5765     scale(0x0);
 5766     disp($off);
 5767   %}
 5768 %}
 5769 
 5770 operand indOffL(iRegP reg, immLoffset off)
 5771 %{
 5772   constraint(ALLOC_IN_RC(ptr_reg));
 5773   match(AddP reg off);
 5774   op_cost(0);
 5775   format %{ "[$reg, $off]" %}
 5776   interface(MEMORY_INTER) %{
 5777     base($reg);
 5778     index(0xffffffff);
 5779     scale(0x0);
 5780     disp($off);
 5781   %}
 5782 %}
 5783 
 5784 operand indOffL1(iRegP reg, immLoffset1 off)
 5785 %{
 5786   constraint(ALLOC_IN_RC(ptr_reg));
 5787   match(AddP reg off);
 5788   op_cost(0);
 5789   format %{ "[$reg, $off]" %}
 5790   interface(MEMORY_INTER) %{
 5791     base($reg);
 5792     index(0xffffffff);
 5793     scale(0x0);
 5794     disp($off);
 5795   %}
 5796 %}
 5797 
 5798 operand indOffL2(iRegP reg, immLoffset2 off)
 5799 %{
 5800   constraint(ALLOC_IN_RC(ptr_reg));
 5801   match(AddP reg off);
 5802   op_cost(0);
 5803   format %{ "[$reg, $off]" %}
 5804   interface(MEMORY_INTER) %{
 5805     base($reg);
 5806     index(0xffffffff);
 5807     scale(0x0);
 5808     disp($off);
 5809   %}
 5810 %}
 5811 
 5812 operand indOffL4(iRegP reg, immLoffset4 off)
 5813 %{
 5814   constraint(ALLOC_IN_RC(ptr_reg));
 5815   match(AddP reg off);
 5816   op_cost(0);
 5817   format %{ "[$reg, $off]" %}
 5818   interface(MEMORY_INTER) %{
 5819     base($reg);
 5820     index(0xffffffff);
 5821     scale(0x0);
 5822     disp($off);
 5823   %}
 5824 %}
 5825 
 5826 operand indOffL8(iRegP reg, immLoffset8 off)
 5827 %{
 5828   constraint(ALLOC_IN_RC(ptr_reg));
 5829   match(AddP reg off);
 5830   op_cost(0);
 5831   format %{ "[$reg, $off]" %}
 5832   interface(MEMORY_INTER) %{
 5833     base($reg);
 5834     index(0xffffffff);
 5835     scale(0x0);
 5836     disp($off);
 5837   %}
 5838 %}
 5839 
 5840 operand indOffL16(iRegP reg, immLoffset16 off)
 5841 %{
 5842   constraint(ALLOC_IN_RC(ptr_reg));
 5843   match(AddP reg off);
 5844   op_cost(0);
 5845   format %{ "[$reg, $off]" %}
 5846   interface(MEMORY_INTER) %{
 5847     base($reg);
 5848     index(0xffffffff);
 5849     scale(0x0);
 5850     disp($off);
 5851   %}
 5852 %}
 5853 
 5854 operand indirectN(iRegN reg)
 5855 %{
 5856   predicate(CompressedOops::shift() == 0);
 5857   constraint(ALLOC_IN_RC(ptr_reg));
 5858   match(DecodeN reg);
 5859   op_cost(0);
 5860   format %{ "[$reg]\t# narrow" %}
 5861   interface(MEMORY_INTER) %{
 5862     base($reg);
 5863     index(0xffffffff);
 5864     scale(0x0);
 5865     disp(0x0);
 5866   %}
 5867 %}
 5868 
 5869 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5870 %{
 5871   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5872   constraint(ALLOC_IN_RC(ptr_reg));
 5873   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5874   op_cost(0);
 5875   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5876   interface(MEMORY_INTER) %{
 5877     base($reg);
 5878     index($ireg);
 5879     scale($scale);
 5880     disp(0x0);
 5881   %}
 5882 %}
 5883 
 5884 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5885 %{
 5886   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5887   constraint(ALLOC_IN_RC(ptr_reg));
 5888   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5889   op_cost(0);
 5890   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5891   interface(MEMORY_INTER) %{
 5892     base($reg);
 5893     index($lreg);
 5894     scale($scale);
 5895     disp(0x0);
 5896   %}
 5897 %}
 5898 
 5899 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5900 %{
 5901   predicate(CompressedOops::shift() == 0);
 5902   constraint(ALLOC_IN_RC(ptr_reg));
 5903   match(AddP (DecodeN reg) (ConvI2L ireg));
 5904   op_cost(0);
 5905   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5906   interface(MEMORY_INTER) %{
 5907     base($reg);
 5908     index($ireg);
 5909     scale(0x0);
 5910     disp(0x0);
 5911   %}
 5912 %}
 5913 
 5914 operand indIndexN(iRegN reg, iRegL lreg)
 5915 %{
 5916   predicate(CompressedOops::shift() == 0);
 5917   constraint(ALLOC_IN_RC(ptr_reg));
 5918   match(AddP (DecodeN reg) lreg);
 5919   op_cost(0);
 5920   format %{ "$reg, $lreg\t# narrow" %}
 5921   interface(MEMORY_INTER) %{
 5922     base($reg);
 5923     index($lreg);
 5924     scale(0x0);
 5925     disp(0x0);
 5926   %}
 5927 %}
 5928 
 5929 operand indOffIN(iRegN reg, immIOffset off)
 5930 %{
 5931   predicate(CompressedOops::shift() == 0);
 5932   constraint(ALLOC_IN_RC(ptr_reg));
 5933   match(AddP (DecodeN reg) off);
 5934   op_cost(0);
 5935   format %{ "[$reg, $off]\t# narrow" %}
 5936   interface(MEMORY_INTER) %{
 5937     base($reg);
 5938     index(0xffffffff);
 5939     scale(0x0);
 5940     disp($off);
 5941   %}
 5942 %}
 5943 
 5944 operand indOffLN(iRegN reg, immLoffset off)
 5945 %{
 5946   predicate(CompressedOops::shift() == 0);
 5947   constraint(ALLOC_IN_RC(ptr_reg));
 5948   match(AddP (DecodeN reg) off);
 5949   op_cost(0);
 5950   format %{ "[$reg, $off]\t# narrow" %}
 5951   interface(MEMORY_INTER) %{
 5952     base($reg);
 5953     index(0xffffffff);
 5954     scale(0x0);
 5955     disp($off);
 5956   %}
 5957 %}
 5958 
 5959 
 5960 
 5961 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 5962 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 5963 %{
 5964   constraint(ALLOC_IN_RC(ptr_reg));
 5965   match(AddP reg off);
 5966   op_cost(0);
 5967   format %{ "[$reg, $off]" %}
 5968   interface(MEMORY_INTER) %{
 5969     base($reg);
 5970     index(0xffffffff);
 5971     scale(0x0);
 5972     disp($off);
 5973   %}
 5974 %}
 5975 
 5976 //----------Special Memory Operands--------------------------------------------
 5977 // Stack Slot Operand - This operand is used for loading and storing temporary
 5978 //                      values on the stack where a match requires a value to
 5979 //                      flow through memory.
 5980 operand stackSlotP(sRegP reg)
 5981 %{
 5982   constraint(ALLOC_IN_RC(stack_slots));
 5983   op_cost(100);
 5984   // No match rule because this operand is only generated in matching
 5985   // match(RegP);
 5986   format %{ "[$reg]" %}
 5987   interface(MEMORY_INTER) %{
 5988     base(0x1e);  // RSP
 5989     index(0x0);  // No Index
 5990     scale(0x0);  // No Scale
 5991     disp($reg);  // Stack Offset
 5992   %}
 5993 %}
 5994 
 5995 operand stackSlotI(sRegI reg)
 5996 %{
 5997   constraint(ALLOC_IN_RC(stack_slots));
 5998   // No match rule because this operand is only generated in matching
 5999   // match(RegI);
 6000   format %{ "[$reg]" %}
 6001   interface(MEMORY_INTER) %{
 6002     base(0x1e);  // RSP
 6003     index(0x0);  // No Index
 6004     scale(0x0);  // No Scale
 6005     disp($reg);  // Stack Offset
 6006   %}
 6007 %}
 6008 
 6009 operand stackSlotF(sRegF reg)
 6010 %{
 6011   constraint(ALLOC_IN_RC(stack_slots));
 6012   // No match rule because this operand is only generated in matching
 6013   // match(RegF);
 6014   format %{ "[$reg]" %}
 6015   interface(MEMORY_INTER) %{
 6016     base(0x1e);  // RSP
 6017     index(0x0);  // No Index
 6018     scale(0x0);  // No Scale
 6019     disp($reg);  // Stack Offset
 6020   %}
 6021 %}
 6022 
 6023 operand stackSlotD(sRegD reg)
 6024 %{
 6025   constraint(ALLOC_IN_RC(stack_slots));
 6026   // No match rule because this operand is only generated in matching
 6027   // match(RegD);
 6028   format %{ "[$reg]" %}
 6029   interface(MEMORY_INTER) %{
 6030     base(0x1e);  // RSP
 6031     index(0x0);  // No Index
 6032     scale(0x0);  // No Scale
 6033     disp($reg);  // Stack Offset
 6034   %}
 6035 %}
 6036 
 6037 operand stackSlotL(sRegL reg)
 6038 %{
 6039   constraint(ALLOC_IN_RC(stack_slots));
 6040   // No match rule because this operand is only generated in matching
 6041   // match(RegL);
 6042   format %{ "[$reg]" %}
 6043   interface(MEMORY_INTER) %{
 6044     base(0x1e);  // RSP
 6045     index(0x0);  // No Index
 6046     scale(0x0);  // No Scale
 6047     disp($reg);  // Stack Offset
 6048   %}
 6049 %}
 6050 
 6051 // Operands for expressing Control Flow
 6052 // NOTE: Label is a predefined operand which should not be redefined in
 6053 //       the AD file. It is generically handled within the ADLC.
 6054 
 6055 //----------Conditional Branch Operands----------------------------------------
 6056 // Comparison Op  - This is the operation of the comparison, and is limited to
 6057 //                  the following set of codes:
 6058 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6059 //
 6060 // Other attributes of the comparison, such as unsignedness, are specified
 6061 // by the comparison instruction that sets a condition code flags register.
 6062 // That result is represented by a flags operand whose subtype is appropriate
 6063 // to the unsignedness (etc.) of the comparison.
 6064 //
 6065 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6066 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6067 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6068 
 6069 // used for signed integral comparisons and fp comparisons
 6070 
 6071 operand cmpOp()
 6072 %{
 6073   match(Bool);
 6074 
 6075   format %{ "" %}
 6076   interface(COND_INTER) %{
 6077     equal(0x0, "eq");
 6078     not_equal(0x1, "ne");
 6079     less(0xb, "lt");
 6080     greater_equal(0xa, "ge");
 6081     less_equal(0xd, "le");
 6082     greater(0xc, "gt");
 6083     overflow(0x6, "vs");
 6084     no_overflow(0x7, "vc");
 6085   %}
 6086 %}
 6087 
 6088 // used for unsigned integral comparisons
 6089 
 6090 operand cmpOpU()
 6091 %{
 6092   match(Bool);
 6093 
 6094   format %{ "" %}
 6095   interface(COND_INTER) %{
 6096     equal(0x0, "eq");
 6097     not_equal(0x1, "ne");
 6098     less(0x3, "lo");
 6099     greater_equal(0x2, "hs");
 6100     less_equal(0x9, "ls");
 6101     greater(0x8, "hi");
 6102     overflow(0x6, "vs");
 6103     no_overflow(0x7, "vc");
 6104   %}
 6105 %}
 6106 
 6107 // used for certain integral comparisons which can be
 6108 // converted to cbxx or tbxx instructions
 6109 
 6110 operand cmpOpEqNe()
 6111 %{
 6112   match(Bool);
 6113   op_cost(0);
 6114   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6115             || n->as_Bool()->_test._test == BoolTest::eq);
 6116 
 6117   format %{ "" %}
 6118   interface(COND_INTER) %{
 6119     equal(0x0, "eq");
 6120     not_equal(0x1, "ne");
 6121     less(0xb, "lt");
 6122     greater_equal(0xa, "ge");
 6123     less_equal(0xd, "le");
 6124     greater(0xc, "gt");
 6125     overflow(0x6, "vs");
 6126     no_overflow(0x7, "vc");
 6127   %}
 6128 %}
 6129 
 6130 // used for certain integral comparisons which can be
 6131 // converted to cbxx or tbxx instructions
 6132 
 6133 operand cmpOpLtGe()
 6134 %{
 6135   match(Bool);
 6136   op_cost(0);
 6137 
 6138   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6139             || n->as_Bool()->_test._test == BoolTest::ge);
 6140 
 6141   format %{ "" %}
 6142   interface(COND_INTER) %{
 6143     equal(0x0, "eq");
 6144     not_equal(0x1, "ne");
 6145     less(0xb, "lt");
 6146     greater_equal(0xa, "ge");
 6147     less_equal(0xd, "le");
 6148     greater(0xc, "gt");
 6149     overflow(0x6, "vs");
 6150     no_overflow(0x7, "vc");
 6151   %}
 6152 %}
 6153 
 6154 // used for certain unsigned integral comparisons which can be
 6155 // converted to cbxx or tbxx instructions
 6156 
 6157 operand cmpOpUEqNeLtGe()
 6158 %{
 6159   match(Bool);
 6160   op_cost(0);
 6161 
 6162   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6163             || n->as_Bool()->_test._test == BoolTest::ne
 6164             || n->as_Bool()->_test._test == BoolTest::lt
 6165             || n->as_Bool()->_test._test == BoolTest::ge);
 6166 
 6167   format %{ "" %}
 6168   interface(COND_INTER) %{
 6169     equal(0x0, "eq");
 6170     not_equal(0x1, "ne");
 6171     less(0xb, "lt");
 6172     greater_equal(0xa, "ge");
 6173     less_equal(0xd, "le");
 6174     greater(0xc, "gt");
 6175     overflow(0x6, "vs");
 6176     no_overflow(0x7, "vc");
 6177   %}
 6178 %}
 6179 
 6180 // Special operand allowing long args to int ops to be truncated for free
 6181 
 6182 operand iRegL2I(iRegL reg) %{
 6183 
 6184   op_cost(0);
 6185 
 6186   match(ConvL2I reg);
 6187 
 6188   format %{ "l2i($reg)" %}
 6189 
 6190   interface(REG_INTER)
 6191 %}
 6192 
 6193 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6194 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6195 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6196 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6197 
 6198 //----------OPERAND CLASSES----------------------------------------------------
 6199 // Operand Classes are groups of operands that are used as to simplify
 6200 // instruction definitions by not requiring the AD writer to specify
 6201 // separate instructions for every form of operand when the
 6202 // instruction accepts multiple operand types with the same basic
 6203 // encoding and format. The classic case of this is memory operands.
 6204 
 6205 // memory is used to define read/write location for load/store
 6206 // instruction defs. we can turn a memory op into an Address
 6207 
 6208 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6209                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6210 
 6211 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6212                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6213 
 6214 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6215                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6216 
 6217 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6218                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6219 
 6220 // All of the memory operands. For the pipeline description.
 6221 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6222                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6223                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6224 
 6225 
 6226 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6227 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6228 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6229 // can be elided because the 32-bit instruction will just employ the
 6230 // lower 32 bits anyway.
 6231 //
 6232 // n.b. this does not elide all L2I conversions. if the truncated
 6233 // value is consumed by more than one operation then the ConvL2I
 6234 // cannot be bundled into the consuming nodes so an l2i gets planted
 6235 // (actually a movw $dst $src) and the downstream instructions consume
 6236 // the result of the l2i as an iRegI input. That's a shame since the
 6237 // movw is actually redundant but its not too costly.
 6238 
 6239 opclass iRegIorL2I(iRegI, iRegL2I);
 6240 
 6241 //----------PIPELINE-----------------------------------------------------------
 6242 // Rules which define the behavior of the target architectures pipeline.
 6243 
 6244 // For specific pipelines, eg A53, define the stages of that pipeline
 6245 //pipe_desc(ISS, EX1, EX2, WR);
 6246 #define ISS S0
 6247 #define EX1 S1
 6248 #define EX2 S2
 6249 #define WR  S3
 6250 
 6251 // Integer ALU reg operation
 6252 pipeline %{
 6253 
 6254 attributes %{
 6255   // ARM instructions are of fixed length
 6256   fixed_size_instructions;        // Fixed size instructions TODO does
 6257   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6258   // ARM instructions come in 32-bit word units
 6259   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6260   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6261   instruction_fetch_units = 1;       // of 64 bytes
 6262 
 6263   // List of nop instructions
 6264   nops( MachNop );
 6265 %}
 6266 
 6267 // We don't use an actual pipeline model so don't care about resources
 6268 // or description. we do use pipeline classes to introduce fixed
 6269 // latencies
 6270 
 6271 //----------RESOURCES----------------------------------------------------------
 6272 // Resources are the functional units available to the machine
 6273 
 6274 resources( INS0, INS1, INS01 = INS0 | INS1,
 6275            ALU0, ALU1, ALU = ALU0 | ALU1,
 6276            MAC,
 6277            DIV,
 6278            BRANCH,
 6279            LDST,
 6280            NEON_FP);
 6281 
 6282 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6283 // Pipeline Description specifies the stages in the machine's pipeline
 6284 
 6285 // Define the pipeline as a generic 6 stage pipeline
 6286 pipe_desc(S0, S1, S2, S3, S4, S5);
 6287 
 6288 //----------PIPELINE CLASSES---------------------------------------------------
 6289 // Pipeline Classes describe the stages in which input and output are
 6290 // referenced by the hardware pipeline.
 6291 
 6292 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6293 %{
 6294   single_instruction;
 6295   src1   : S1(read);
 6296   src2   : S2(read);
 6297   dst    : S5(write);
 6298   INS01  : ISS;
 6299   NEON_FP : S5;
 6300 %}
 6301 
 6302 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6303 %{
 6304   single_instruction;
 6305   src1   : S1(read);
 6306   src2   : S2(read);
 6307   dst    : S5(write);
 6308   INS01  : ISS;
 6309   NEON_FP : S5;
 6310 %}
 6311 
 6312 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6313 %{
 6314   single_instruction;
 6315   src    : S1(read);
 6316   dst    : S5(write);
 6317   INS01  : ISS;
 6318   NEON_FP : S5;
 6319 %}
 6320 
 6321 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6322 %{
 6323   single_instruction;
 6324   src    : S1(read);
 6325   dst    : S5(write);
 6326   INS01  : ISS;
 6327   NEON_FP : S5;
 6328 %}
 6329 
 6330 pipe_class fp_d2f(vRegF dst, vRegD src)
 6331 %{
 6332   single_instruction;
 6333   src    : S1(read);
 6334   dst    : S5(write);
 6335   INS01  : ISS;
 6336   NEON_FP : S5;
 6337 %}
 6338 
 6339 pipe_class fp_f2d(vRegD dst, vRegF src)
 6340 %{
 6341   single_instruction;
 6342   src    : S1(read);
 6343   dst    : S5(write);
 6344   INS01  : ISS;
 6345   NEON_FP : S5;
 6346 %}
 6347 
 6348 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6349 %{
 6350   single_instruction;
 6351   src    : S1(read);
 6352   dst    : S5(write);
 6353   INS01  : ISS;
 6354   NEON_FP : S5;
 6355 %}
 6356 
 6357 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6358 %{
 6359   single_instruction;
 6360   src    : S1(read);
 6361   dst    : S5(write);
 6362   INS01  : ISS;
 6363   NEON_FP : S5;
 6364 %}
 6365 
 6366 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6367 %{
 6368   single_instruction;
 6369   src    : S1(read);
 6370   dst    : S5(write);
 6371   INS01  : ISS;
 6372   NEON_FP : S5;
 6373 %}
 6374 
 6375 pipe_class fp_l2f(vRegF dst, iRegL src)
 6376 %{
 6377   single_instruction;
 6378   src    : S1(read);
 6379   dst    : S5(write);
 6380   INS01  : ISS;
 6381   NEON_FP : S5;
 6382 %}
 6383 
 6384 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6385 %{
 6386   single_instruction;
 6387   src    : S1(read);
 6388   dst    : S5(write);
 6389   INS01  : ISS;
 6390   NEON_FP : S5;
 6391 %}
 6392 
 6393 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6394 %{
 6395   single_instruction;
 6396   src    : S1(read);
 6397   dst    : S5(write);
 6398   INS01  : ISS;
 6399   NEON_FP : S5;
 6400 %}
 6401 
 6402 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6403 %{
 6404   single_instruction;
 6405   src    : S1(read);
 6406   dst    : S5(write);
 6407   INS01  : ISS;
 6408   NEON_FP : S5;
 6409 %}
 6410 
 6411 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6412 %{
 6413   single_instruction;
 6414   src    : S1(read);
 6415   dst    : S5(write);
 6416   INS01  : ISS;
 6417   NEON_FP : S5;
 6418 %}
 6419 
 6420 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6421 %{
 6422   single_instruction;
 6423   src1   : S1(read);
 6424   src2   : S2(read);
 6425   dst    : S5(write);
 6426   INS0   : ISS;
 6427   NEON_FP : S5;
 6428 %}
 6429 
 6430 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6431 %{
 6432   single_instruction;
 6433   src1   : S1(read);
 6434   src2   : S2(read);
 6435   dst    : S5(write);
 6436   INS0   : ISS;
 6437   NEON_FP : S5;
 6438 %}
 6439 
 6440 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6441 %{
 6442   single_instruction;
 6443   cr     : S1(read);
 6444   src1   : S1(read);
 6445   src2   : S1(read);
 6446   dst    : S3(write);
 6447   INS01  : ISS;
 6448   NEON_FP : S3;
 6449 %}
 6450 
 6451 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6452 %{
 6453   single_instruction;
 6454   cr     : S1(read);
 6455   src1   : S1(read);
 6456   src2   : S1(read);
 6457   dst    : S3(write);
 6458   INS01  : ISS;
 6459   NEON_FP : S3;
 6460 %}
 6461 
 6462 pipe_class fp_imm_s(vRegF dst)
 6463 %{
 6464   single_instruction;
 6465   dst    : S3(write);
 6466   INS01  : ISS;
 6467   NEON_FP : S3;
 6468 %}
 6469 
 6470 pipe_class fp_imm_d(vRegD dst)
 6471 %{
 6472   single_instruction;
 6473   dst    : S3(write);
 6474   INS01  : ISS;
 6475   NEON_FP : S3;
 6476 %}
 6477 
 6478 pipe_class fp_load_constant_s(vRegF dst)
 6479 %{
 6480   single_instruction;
 6481   dst    : S4(write);
 6482   INS01  : ISS;
 6483   NEON_FP : S4;
 6484 %}
 6485 
 6486 pipe_class fp_load_constant_d(vRegD dst)
 6487 %{
 6488   single_instruction;
 6489   dst    : S4(write);
 6490   INS01  : ISS;
 6491   NEON_FP : S4;
 6492 %}
 6493 
 6494 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6495 %{
 6496   single_instruction;
 6497   dst    : S5(write);
 6498   src1   : S1(read);
 6499   src2   : S1(read);
 6500   INS01  : ISS;
 6501   NEON_FP : S5;
 6502 %}
 6503 
 6504 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6505 %{
 6506   single_instruction;
 6507   dst    : S5(write);
 6508   src1   : S1(read);
 6509   src2   : S1(read);
 6510   INS0   : ISS;
 6511   NEON_FP : S5;
 6512 %}
 6513 
 6514 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6515 %{
 6516   single_instruction;
 6517   dst    : S5(write);
 6518   src1   : S1(read);
 6519   src2   : S1(read);
 6520   dst    : S1(read);
 6521   INS01  : ISS;
 6522   NEON_FP : S5;
 6523 %}
 6524 
 6525 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6526 %{
 6527   single_instruction;
 6528   dst    : S5(write);
 6529   src1   : S1(read);
 6530   src2   : S1(read);
 6531   dst    : S1(read);
 6532   INS0   : ISS;
 6533   NEON_FP : S5;
 6534 %}
 6535 
 6536 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6537 %{
 6538   single_instruction;
 6539   dst    : S4(write);
 6540   src1   : S2(read);
 6541   src2   : S2(read);
 6542   INS01  : ISS;
 6543   NEON_FP : S4;
 6544 %}
 6545 
 6546 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6547 %{
 6548   single_instruction;
 6549   dst    : S4(write);
 6550   src1   : S2(read);
 6551   src2   : S2(read);
 6552   INS0   : ISS;
 6553   NEON_FP : S4;
 6554 %}
 6555 
 6556 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6557 %{
 6558   single_instruction;
 6559   dst    : S3(write);
 6560   src1   : S2(read);
 6561   src2   : S2(read);
 6562   INS01  : ISS;
 6563   NEON_FP : S3;
 6564 %}
 6565 
 6566 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6567 %{
 6568   single_instruction;
 6569   dst    : S3(write);
 6570   src1   : S2(read);
 6571   src2   : S2(read);
 6572   INS0   : ISS;
 6573   NEON_FP : S3;
 6574 %}
 6575 
 6576 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6577 %{
 6578   single_instruction;
 6579   dst    : S3(write);
 6580   src    : S1(read);
 6581   shift  : S1(read);
 6582   INS01  : ISS;
 6583   NEON_FP : S3;
 6584 %}
 6585 
 6586 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6587 %{
 6588   single_instruction;
 6589   dst    : S3(write);
 6590   src    : S1(read);
 6591   shift  : S1(read);
 6592   INS0   : ISS;
 6593   NEON_FP : S3;
 6594 %}
 6595 
 6596 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6597 %{
 6598   single_instruction;
 6599   dst    : S3(write);
 6600   src    : S1(read);
 6601   INS01  : ISS;
 6602   NEON_FP : S3;
 6603 %}
 6604 
 6605 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6606 %{
 6607   single_instruction;
 6608   dst    : S3(write);
 6609   src    : S1(read);
 6610   INS0   : ISS;
 6611   NEON_FP : S3;
 6612 %}
 6613 
 6614 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6615 %{
 6616   single_instruction;
 6617   dst    : S5(write);
 6618   src1   : S1(read);
 6619   src2   : S1(read);
 6620   INS01  : ISS;
 6621   NEON_FP : S5;
 6622 %}
 6623 
 6624 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6625 %{
 6626   single_instruction;
 6627   dst    : S5(write);
 6628   src1   : S1(read);
 6629   src2   : S1(read);
 6630   INS0   : ISS;
 6631   NEON_FP : S5;
 6632 %}
 6633 
 6634 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6635 %{
 6636   single_instruction;
 6637   dst    : S5(write);
 6638   src1   : S1(read);
 6639   src2   : S1(read);
 6640   INS0   : ISS;
 6641   NEON_FP : S5;
 6642 %}
 6643 
 6644 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6645 %{
 6646   single_instruction;
 6647   dst    : S5(write);
 6648   src1   : S1(read);
 6649   src2   : S1(read);
 6650   INS0   : ISS;
 6651   NEON_FP : S5;
 6652 %}
 6653 
 6654 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6655 %{
 6656   single_instruction;
 6657   dst    : S5(write);
 6658   src    : S1(read);
 6659   INS0   : ISS;
 6660   NEON_FP : S5;
 6661 %}
 6662 
 6663 pipe_class vunop_fp64(vecD dst, vecD src)
 6664 %{
 6665   single_instruction;
 6666   dst    : S5(write);
 6667   src    : S1(read);
 6668   INS01  : ISS;
 6669   NEON_FP : S5;
 6670 %}
 6671 
 6672 pipe_class vunop_fp128(vecX dst, vecX src)
 6673 %{
 6674   single_instruction;
 6675   dst    : S5(write);
 6676   src    : S1(read);
 6677   INS0   : ISS;
 6678   NEON_FP : S5;
 6679 %}
 6680 
 6681 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6682 %{
 6683   single_instruction;
 6684   dst    : S3(write);
 6685   src    : S1(read);
 6686   INS01  : ISS;
 6687   NEON_FP : S3;
 6688 %}
 6689 
 6690 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6691 %{
 6692   single_instruction;
 6693   dst    : S3(write);
 6694   src    : S1(read);
 6695   INS01  : ISS;
 6696   NEON_FP : S3;
 6697 %}
 6698 
 6699 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6700 %{
 6701   single_instruction;
 6702   dst    : S3(write);
 6703   src    : S1(read);
 6704   INS01  : ISS;
 6705   NEON_FP : S3;
 6706 %}
 6707 
 6708 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6709 %{
 6710   single_instruction;
 6711   dst    : S3(write);
 6712   src    : S1(read);
 6713   INS01  : ISS;
 6714   NEON_FP : S3;
 6715 %}
 6716 
 6717 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6718 %{
 6719   single_instruction;
 6720   dst    : S3(write);
 6721   src    : S1(read);
 6722   INS01  : ISS;
 6723   NEON_FP : S3;
 6724 %}
 6725 
 6726 pipe_class vmovi_reg_imm64(vecD dst)
 6727 %{
 6728   single_instruction;
 6729   dst    : S3(write);
 6730   INS01  : ISS;
 6731   NEON_FP : S3;
 6732 %}
 6733 
 6734 pipe_class vmovi_reg_imm128(vecX dst)
 6735 %{
 6736   single_instruction;
 6737   dst    : S3(write);
 6738   INS0   : ISS;
 6739   NEON_FP : S3;
 6740 %}
 6741 
 6742 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6743 %{
 6744   single_instruction;
 6745   dst    : S5(write);
 6746   mem    : ISS(read);
 6747   INS01  : ISS;
 6748   NEON_FP : S3;
 6749 %}
 6750 
 6751 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6752 %{
 6753   single_instruction;
 6754   dst    : S5(write);
 6755   mem    : ISS(read);
 6756   INS01  : ISS;
 6757   NEON_FP : S3;
 6758 %}
 6759 
 6760 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6761 %{
 6762   single_instruction;
 6763   mem    : ISS(read);
 6764   src    : S2(read);
 6765   INS01  : ISS;
 6766   NEON_FP : S3;
 6767 %}
 6768 
 6769 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6770 %{
 6771   single_instruction;
 6772   mem    : ISS(read);
 6773   src    : S2(read);
 6774   INS01  : ISS;
 6775   NEON_FP : S3;
 6776 %}
 6777 
 6778 //------- Integer ALU operations --------------------------
 6779 
 6780 // Integer ALU reg-reg operation
 6781 // Operands needed in EX1, result generated in EX2
 6782 // Eg.  ADD     x0, x1, x2
 6783 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6784 %{
 6785   single_instruction;
 6786   dst    : EX2(write);
 6787   src1   : EX1(read);
 6788   src2   : EX1(read);
 6789   INS01  : ISS; // Dual issue as instruction 0 or 1
 6790   ALU    : EX2;
 6791 %}
 6792 
 6793 // Integer ALU reg-reg operation with constant shift
 6794 // Shifted register must be available in LATE_ISS instead of EX1
 6795 // Eg.  ADD     x0, x1, x2, LSL #2
 6796 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6797 %{
 6798   single_instruction;
 6799   dst    : EX2(write);
 6800   src1   : EX1(read);
 6801   src2   : ISS(read);
 6802   INS01  : ISS;
 6803   ALU    : EX2;
 6804 %}
 6805 
 6806 // Integer ALU reg operation with constant shift
 6807 // Eg.  LSL     x0, x1, #shift
 6808 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6809 %{
 6810   single_instruction;
 6811   dst    : EX2(write);
 6812   src1   : ISS(read);
 6813   INS01  : ISS;
 6814   ALU    : EX2;
 6815 %}
 6816 
 6817 // Integer ALU reg-reg operation with variable shift
 6818 // Both operands must be available in LATE_ISS instead of EX1
 6819 // Result is available in EX1 instead of EX2
 6820 // Eg.  LSLV    x0, x1, x2
 6821 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6822 %{
 6823   single_instruction;
 6824   dst    : EX1(write);
 6825   src1   : ISS(read);
 6826   src2   : ISS(read);
 6827   INS01  : ISS;
 6828   ALU    : EX1;
 6829 %}
 6830 
 6831 // Integer ALU reg-reg operation with extract
 6832 // As for _vshift above, but result generated in EX2
 6833 // Eg.  EXTR    x0, x1, x2, #N
 6834 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6835 %{
 6836   single_instruction;
 6837   dst    : EX2(write);
 6838   src1   : ISS(read);
 6839   src2   : ISS(read);
 6840   INS1   : ISS; // Can only dual issue as Instruction 1
 6841   ALU    : EX1;
 6842 %}
 6843 
 6844 // Integer ALU reg operation
 6845 // Eg.  NEG     x0, x1
 6846 pipe_class ialu_reg(iRegI dst, iRegI src)
 6847 %{
 6848   single_instruction;
 6849   dst    : EX2(write);
 6850   src    : EX1(read);
 6851   INS01  : ISS;
 6852   ALU    : EX2;
 6853 %}
 6854 
 6855 // Integer ALU reg mmediate operation
 6856 // Eg.  ADD     x0, x1, #N
 6857 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6858 %{
 6859   single_instruction;
 6860   dst    : EX2(write);
 6861   src1   : EX1(read);
 6862   INS01  : ISS;
 6863   ALU    : EX2;
 6864 %}
 6865 
 6866 // Integer ALU immediate operation (no source operands)
 6867 // Eg.  MOV     x0, #N
 6868 pipe_class ialu_imm(iRegI dst)
 6869 %{
 6870   single_instruction;
 6871   dst    : EX1(write);
 6872   INS01  : ISS;
 6873   ALU    : EX1;
 6874 %}
 6875 
 6876 //------- Compare operation -------------------------------
 6877 
 6878 // Compare reg-reg
 6879 // Eg.  CMP     x0, x1
 6880 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6881 %{
 6882   single_instruction;
 6883 //  fixed_latency(16);
 6884   cr     : EX2(write);
 6885   op1    : EX1(read);
 6886   op2    : EX1(read);
 6887   INS01  : ISS;
 6888   ALU    : EX2;
 6889 %}
 6890 
 6891 // Compare reg-reg
 6892 // Eg.  CMP     x0, #N
 6893 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6894 %{
 6895   single_instruction;
 6896 //  fixed_latency(16);
 6897   cr     : EX2(write);
 6898   op1    : EX1(read);
 6899   INS01  : ISS;
 6900   ALU    : EX2;
 6901 %}
 6902 
 6903 //------- Conditional instructions ------------------------
 6904 
 6905 // Conditional no operands
 6906 // Eg.  CSINC   x0, zr, zr, <cond>
 6907 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6908 %{
 6909   single_instruction;
 6910   cr     : EX1(read);
 6911   dst    : EX2(write);
 6912   INS01  : ISS;
 6913   ALU    : EX2;
 6914 %}
 6915 
 6916 // Conditional 2 operand
 6917 // EG.  CSEL    X0, X1, X2, <cond>
 6918 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6919 %{
 6920   single_instruction;
 6921   cr     : EX1(read);
 6922   src1   : EX1(read);
 6923   src2   : EX1(read);
 6924   dst    : EX2(write);
 6925   INS01  : ISS;
 6926   ALU    : EX2;
 6927 %}
 6928 
 6929 // Conditional 2 operand
 6930 // EG.  CSEL    X0, X1, X2, <cond>
 6931 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6932 %{
 6933   single_instruction;
 6934   cr     : EX1(read);
 6935   src    : EX1(read);
 6936   dst    : EX2(write);
 6937   INS01  : ISS;
 6938   ALU    : EX2;
 6939 %}
 6940 
 6941 //------- Multiply pipeline operations --------------------
 6942 
 6943 // Multiply reg-reg
 6944 // Eg.  MUL     w0, w1, w2
 6945 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6946 %{
 6947   single_instruction;
 6948   dst    : WR(write);
 6949   src1   : ISS(read);
 6950   src2   : ISS(read);
 6951   INS01  : ISS;
 6952   MAC    : WR;
 6953 %}
 6954 
 6955 // Multiply accumulate
 6956 // Eg.  MADD    w0, w1, w2, w3
 6957 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6958 %{
 6959   single_instruction;
 6960   dst    : WR(write);
 6961   src1   : ISS(read);
 6962   src2   : ISS(read);
 6963   src3   : ISS(read);
 6964   INS01  : ISS;
 6965   MAC    : WR;
 6966 %}
 6967 
 6968 // Eg.  MUL     w0, w1, w2
 6969 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6970 %{
 6971   single_instruction;
 6972   fixed_latency(3); // Maximum latency for 64 bit mul
 6973   dst    : WR(write);
 6974   src1   : ISS(read);
 6975   src2   : ISS(read);
 6976   INS01  : ISS;
 6977   MAC    : WR;
 6978 %}
 6979 
 6980 // Multiply accumulate
 6981 // Eg.  MADD    w0, w1, w2, w3
 6982 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6983 %{
 6984   single_instruction;
 6985   fixed_latency(3); // Maximum latency for 64 bit mul
 6986   dst    : WR(write);
 6987   src1   : ISS(read);
 6988   src2   : ISS(read);
 6989   src3   : ISS(read);
 6990   INS01  : ISS;
 6991   MAC    : WR;
 6992 %}
 6993 
 6994 //------- Divide pipeline operations --------------------
 6995 
 6996 // Eg.  SDIV    w0, w1, w2
 6997 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6998 %{
 6999   single_instruction;
 7000   fixed_latency(8); // Maximum latency for 32 bit divide
 7001   dst    : WR(write);
 7002   src1   : ISS(read);
 7003   src2   : ISS(read);
 7004   INS0   : ISS; // Can only dual issue as instruction 0
 7005   DIV    : WR;
 7006 %}
 7007 
 7008 // Eg.  SDIV    x0, x1, x2
 7009 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7010 %{
 7011   single_instruction;
 7012   fixed_latency(16); // Maximum latency for 64 bit divide
 7013   dst    : WR(write);
 7014   src1   : ISS(read);
 7015   src2   : ISS(read);
 7016   INS0   : ISS; // Can only dual issue as instruction 0
 7017   DIV    : WR;
 7018 %}
 7019 
 7020 //------- Load pipeline operations ------------------------
 7021 
 7022 // Load - prefetch
 7023 // Eg.  PFRM    <mem>
 7024 pipe_class iload_prefetch(memory mem)
 7025 %{
 7026   single_instruction;
 7027   mem    : ISS(read);
 7028   INS01  : ISS;
 7029   LDST   : WR;
 7030 %}
 7031 
 7032 // Load - reg, mem
 7033 // Eg.  LDR     x0, <mem>
 7034 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7035 %{
 7036   single_instruction;
 7037   dst    : WR(write);
 7038   mem    : ISS(read);
 7039   INS01  : ISS;
 7040   LDST   : WR;
 7041 %}
 7042 
 7043 // Load - reg, reg
 7044 // Eg.  LDR     x0, [sp, x1]
 7045 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7046 %{
 7047   single_instruction;
 7048   dst    : WR(write);
 7049   src    : ISS(read);
 7050   INS01  : ISS;
 7051   LDST   : WR;
 7052 %}
 7053 
 7054 //------- Store pipeline operations -----------------------
 7055 
 7056 // Store - zr, mem
 7057 // Eg.  STR     zr, <mem>
 7058 pipe_class istore_mem(memory mem)
 7059 %{
 7060   single_instruction;
 7061   mem    : ISS(read);
 7062   INS01  : ISS;
 7063   LDST   : WR;
 7064 %}
 7065 
 7066 // Store - reg, mem
 7067 // Eg.  STR     x0, <mem>
 7068 pipe_class istore_reg_mem(iRegI src, memory mem)
 7069 %{
 7070   single_instruction;
 7071   mem    : ISS(read);
 7072   src    : EX2(read);
 7073   INS01  : ISS;
 7074   LDST   : WR;
 7075 %}
 7076 
 7077 // Store - reg, reg
 7078 // Eg. STR      x0, [sp, x1]
 7079 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7080 %{
 7081   single_instruction;
 7082   dst    : ISS(read);
 7083   src    : EX2(read);
 7084   INS01  : ISS;
 7085   LDST   : WR;
 7086 %}
 7087 
 7088 //------- Store pipeline operations -----------------------
 7089 
 7090 // Branch
 7091 pipe_class pipe_branch()
 7092 %{
 7093   single_instruction;
 7094   INS01  : ISS;
 7095   BRANCH : EX1;
 7096 %}
 7097 
 7098 // Conditional branch
 7099 pipe_class pipe_branch_cond(rFlagsReg cr)
 7100 %{
 7101   single_instruction;
 7102   cr     : EX1(read);
 7103   INS01  : ISS;
 7104   BRANCH : EX1;
 7105 %}
 7106 
 7107 // Compare & Branch
 7108 // EG.  CBZ/CBNZ
 7109 pipe_class pipe_cmp_branch(iRegI op1)
 7110 %{
 7111   single_instruction;
 7112   op1    : EX1(read);
 7113   INS01  : ISS;
 7114   BRANCH : EX1;
 7115 %}
 7116 
 7117 //------- Synchronisation operations ----------------------
 7118 
 7119 // Any operation requiring serialization.
 7120 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7121 pipe_class pipe_serial()
 7122 %{
 7123   single_instruction;
 7124   force_serialization;
 7125   fixed_latency(16);
 7126   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7127   LDST   : WR;
 7128 %}
 7129 
 7130 // Generic big/slow expanded idiom - also serialized
 7131 pipe_class pipe_slow()
 7132 %{
 7133   instruction_count(10);
 7134   multiple_bundles;
 7135   force_serialization;
 7136   fixed_latency(16);
 7137   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7138   LDST   : WR;
 7139 %}
 7140 
 7141 // Empty pipeline class
 7142 pipe_class pipe_class_empty()
 7143 %{
 7144   single_instruction;
 7145   fixed_latency(0);
 7146 %}
 7147 
 7148 // Default pipeline class.
 7149 pipe_class pipe_class_default()
 7150 %{
 7151   single_instruction;
 7152   fixed_latency(2);
 7153 %}
 7154 
 7155 // Pipeline class for compares.
 7156 pipe_class pipe_class_compare()
 7157 %{
 7158   single_instruction;
 7159   fixed_latency(16);
 7160 %}
 7161 
 7162 // Pipeline class for memory operations.
 7163 pipe_class pipe_class_memory()
 7164 %{
 7165   single_instruction;
 7166   fixed_latency(16);
 7167 %}
 7168 
 7169 // Pipeline class for call.
 7170 pipe_class pipe_class_call()
 7171 %{
 7172   single_instruction;
 7173   fixed_latency(100);
 7174 %}
 7175 
 7176 // Define the class for the Nop node.
 7177 define %{
 7178    MachNop = pipe_class_empty;
 7179 %}
 7180 
 7181 %}
 7182 //----------INSTRUCTIONS-------------------------------------------------------
 7183 //
 7184 // match      -- States which machine-independent subtree may be replaced
 7185 //               by this instruction.
 7186 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7187 //               selection to identify a minimum cost tree of machine
 7188 //               instructions that matches a tree of machine-independent
 7189 //               instructions.
 7190 // format     -- A string providing the disassembly for this instruction.
 7191 //               The value of an instruction's operand may be inserted
 7192 //               by referring to it with a '$' prefix.
 7193 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7194 //               to within an encode class as $primary, $secondary, and $tertiary
 7195 //               rrspectively.  The primary opcode is commonly used to
 7196 //               indicate the type of machine instruction, while secondary
 7197 //               and tertiary are often used for prefix options or addressing
 7198 //               modes.
 7199 // ins_encode -- A list of encode classes with parameters. The encode class
 7200 //               name must have been defined in an 'enc_class' specification
 7201 //               in the encode section of the architecture description.
 7202 
 7203 // ============================================================================
 7204 // Memory (Load/Store) Instructions
 7205 
 7206 // Load Instructions
 7207 
 7208 // Load Byte (8 bit signed)
 7209 instruct loadB(iRegINoSp dst, memory1 mem)
 7210 %{
 7211   match(Set dst (LoadB mem));
 7212   predicate(!needs_acquiring_load(n));
 7213 
 7214   ins_cost(4 * INSN_COST);
 7215   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7216 
 7217   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7218 
 7219   ins_pipe(iload_reg_mem);
 7220 %}
 7221 
 7222 // Load Byte (8 bit signed) into long
 7223 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7224 %{
 7225   match(Set dst (ConvI2L (LoadB mem)));
 7226   predicate(!needs_acquiring_load(n->in(1)));
 7227 
 7228   ins_cost(4 * INSN_COST);
 7229   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7230 
 7231   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7232 
 7233   ins_pipe(iload_reg_mem);
 7234 %}
 7235 
 7236 // Load Byte (8 bit unsigned)
 7237 instruct loadUB(iRegINoSp dst, memory1 mem)
 7238 %{
 7239   match(Set dst (LoadUB mem));
 7240   predicate(!needs_acquiring_load(n));
 7241 
 7242   ins_cost(4 * INSN_COST);
 7243   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7244 
 7245   ins_encode(aarch64_enc_ldrb(dst, mem));
 7246 
 7247   ins_pipe(iload_reg_mem);
 7248 %}
 7249 
 7250 // Load Byte (8 bit unsigned) into long
 7251 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7252 %{
 7253   match(Set dst (ConvI2L (LoadUB mem)));
 7254   predicate(!needs_acquiring_load(n->in(1)));
 7255 
 7256   ins_cost(4 * INSN_COST);
 7257   format %{ "ldrb  $dst, $mem\t# byte" %}
 7258 
 7259   ins_encode(aarch64_enc_ldrb(dst, mem));
 7260 
 7261   ins_pipe(iload_reg_mem);
 7262 %}
 7263 
 7264 // Load Short (16 bit signed)
 7265 instruct loadS(iRegINoSp dst, memory2 mem)
 7266 %{
 7267   match(Set dst (LoadS mem));
 7268   predicate(!needs_acquiring_load(n));
 7269 
 7270   ins_cost(4 * INSN_COST);
 7271   format %{ "ldrshw  $dst, $mem\t# short" %}
 7272 
 7273   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7274 
 7275   ins_pipe(iload_reg_mem);
 7276 %}
 7277 
 7278 // Load Short (16 bit signed) into long
 7279 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7280 %{
 7281   match(Set dst (ConvI2L (LoadS mem)));
 7282   predicate(!needs_acquiring_load(n->in(1)));
 7283 
 7284   ins_cost(4 * INSN_COST);
 7285   format %{ "ldrsh  $dst, $mem\t# short" %}
 7286 
 7287   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7288 
 7289   ins_pipe(iload_reg_mem);
 7290 %}
 7291 
 7292 // Load Char (16 bit unsigned)
 7293 instruct loadUS(iRegINoSp dst, memory2 mem)
 7294 %{
 7295   match(Set dst (LoadUS mem));
 7296   predicate(!needs_acquiring_load(n));
 7297 
 7298   ins_cost(4 * INSN_COST);
 7299   format %{ "ldrh  $dst, $mem\t# short" %}
 7300 
 7301   ins_encode(aarch64_enc_ldrh(dst, mem));
 7302 
 7303   ins_pipe(iload_reg_mem);
 7304 %}
 7305 
 7306 // Load Short/Char (16 bit unsigned) into long
 7307 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7308 %{
 7309   match(Set dst (ConvI2L (LoadUS mem)));
 7310   predicate(!needs_acquiring_load(n->in(1)));
 7311 
 7312   ins_cost(4 * INSN_COST);
 7313   format %{ "ldrh  $dst, $mem\t# short" %}
 7314 
 7315   ins_encode(aarch64_enc_ldrh(dst, mem));
 7316 
 7317   ins_pipe(iload_reg_mem);
 7318 %}
 7319 
 7320 // Load Integer (32 bit signed)
 7321 instruct loadI(iRegINoSp dst, memory4 mem)
 7322 %{
 7323   match(Set dst (LoadI mem));
 7324   predicate(!needs_acquiring_load(n));
 7325 
 7326   ins_cost(4 * INSN_COST);
 7327   format %{ "ldrw  $dst, $mem\t# int" %}
 7328 
 7329   ins_encode(aarch64_enc_ldrw(dst, mem));
 7330 
 7331   ins_pipe(iload_reg_mem);
 7332 %}
 7333 
 7334 // Load Integer (32 bit signed) into long
 7335 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7336 %{
 7337   match(Set dst (ConvI2L (LoadI mem)));
 7338   predicate(!needs_acquiring_load(n->in(1)));
 7339 
 7340   ins_cost(4 * INSN_COST);
 7341   format %{ "ldrsw  $dst, $mem\t# int" %}
 7342 
 7343   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7344 
 7345   ins_pipe(iload_reg_mem);
 7346 %}
 7347 
 7348 // Load Integer (32 bit unsigned) into long
 7349 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7350 %{
 7351   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7352   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7353 
 7354   ins_cost(4 * INSN_COST);
 7355   format %{ "ldrw  $dst, $mem\t# int" %}
 7356 
 7357   ins_encode(aarch64_enc_ldrw(dst, mem));
 7358 
 7359   ins_pipe(iload_reg_mem);
 7360 %}
 7361 
 7362 // Load Long (64 bit signed)
 7363 instruct loadL(iRegLNoSp dst, memory8 mem)
 7364 %{
 7365   match(Set dst (LoadL mem));
 7366   predicate(!needs_acquiring_load(n));
 7367 
 7368   ins_cost(4 * INSN_COST);
 7369   format %{ "ldr  $dst, $mem\t# int" %}
 7370 
 7371   ins_encode(aarch64_enc_ldr(dst, mem));
 7372 
 7373   ins_pipe(iload_reg_mem);
 7374 %}
 7375 
 7376 // Load Range
 7377 instruct loadRange(iRegINoSp dst, memory4 mem)
 7378 %{
 7379   match(Set dst (LoadRange mem));
 7380 
 7381   ins_cost(4 * INSN_COST);
 7382   format %{ "ldrw  $dst, $mem\t# range" %}
 7383 
 7384   ins_encode(aarch64_enc_ldrw(dst, mem));
 7385 
 7386   ins_pipe(iload_reg_mem);
 7387 %}
 7388 
 7389 // Load Pointer
 7390 instruct loadP(iRegPNoSp dst, memory8 mem)
 7391 %{
 7392   match(Set dst (LoadP mem));
 7393   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7394 
 7395   ins_cost(4 * INSN_COST);
 7396   format %{ "ldr  $dst, $mem\t# ptr" %}
 7397 
 7398   ins_encode(aarch64_enc_ldr(dst, mem));
 7399 
 7400   ins_pipe(iload_reg_mem);
 7401 %}
 7402 
 7403 // Load Compressed Pointer
 7404 instruct loadN(iRegNNoSp dst, memory4 mem)
 7405 %{
 7406   match(Set dst (LoadN mem));
 7407   predicate(!needs_acquiring_load(n));
 7408 
 7409   ins_cost(4 * INSN_COST);
 7410   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7411 
 7412   ins_encode(aarch64_enc_ldrw(dst, mem));
 7413 
 7414   ins_pipe(iload_reg_mem);
 7415 %}
 7416 
 7417 // Load Klass Pointer
 7418 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7419 %{
 7420   match(Set dst (LoadKlass mem));
 7421   predicate(!needs_acquiring_load(n));
 7422 
 7423   ins_cost(4 * INSN_COST);
 7424   format %{ "ldr  $dst, $mem\t# class" %}
 7425 
 7426   ins_encode(aarch64_enc_ldr(dst, mem));
 7427 
 7428   ins_pipe(iload_reg_mem);
 7429 %}
 7430 
 7431 // Load Narrow Klass Pointer
 7432 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7433 %{
 7434   match(Set dst (LoadNKlass mem));
 7435   predicate(!needs_acquiring_load(n));
 7436 
 7437   ins_cost(4 * INSN_COST);
 7438   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7439 
 7440   ins_encode(aarch64_enc_ldrw(dst, mem));
 7441 
 7442   ins_pipe(iload_reg_mem);
 7443 %}
 7444 
 7445 // Load Float
 7446 instruct loadF(vRegF dst, memory4 mem)
 7447 %{
 7448   match(Set dst (LoadF mem));
 7449   predicate(!needs_acquiring_load(n));
 7450 
 7451   ins_cost(4 * INSN_COST);
 7452   format %{ "ldrs  $dst, $mem\t# float" %}
 7453 
 7454   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7455 
 7456   ins_pipe(pipe_class_memory);
 7457 %}
 7458 
 7459 // Load Double
 7460 instruct loadD(vRegD dst, memory8 mem)
 7461 %{
 7462   match(Set dst (LoadD mem));
 7463   predicate(!needs_acquiring_load(n));
 7464 
 7465   ins_cost(4 * INSN_COST);
 7466   format %{ "ldrd  $dst, $mem\t# double" %}
 7467 
 7468   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7469 
 7470   ins_pipe(pipe_class_memory);
 7471 %}
 7472 
 7473 
 7474 // Load Int Constant
 7475 instruct loadConI(iRegINoSp dst, immI src)
 7476 %{
 7477   match(Set dst src);
 7478 
 7479   ins_cost(INSN_COST);
 7480   format %{ "mov $dst, $src\t# int" %}
 7481 
 7482   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7483 
 7484   ins_pipe(ialu_imm);
 7485 %}
 7486 
 7487 // Load Long Constant
 7488 instruct loadConL(iRegLNoSp dst, immL src)
 7489 %{
 7490   match(Set dst src);
 7491 
 7492   ins_cost(INSN_COST);
 7493   format %{ "mov $dst, $src\t# long" %}
 7494 
 7495   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7496 
 7497   ins_pipe(ialu_imm);
 7498 %}
 7499 
 7500 // Load Pointer Constant
 7501 
 7502 instruct loadConP(iRegPNoSp dst, immP con)
 7503 %{
 7504   match(Set dst con);
 7505 
 7506   ins_cost(INSN_COST * 4);
 7507   format %{
 7508     "mov  $dst, $con\t# ptr\n\t"
 7509   %}
 7510 
 7511   ins_encode(aarch64_enc_mov_p(dst, con));
 7512 
 7513   ins_pipe(ialu_imm);
 7514 %}
 7515 
 7516 // Load Null Pointer Constant
 7517 
 7518 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7519 %{
 7520   match(Set dst con);
 7521 
 7522   ins_cost(INSN_COST);
 7523   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7524 
 7525   ins_encode(aarch64_enc_mov_p0(dst, con));
 7526 
 7527   ins_pipe(ialu_imm);
 7528 %}
 7529 
 7530 // Load Pointer Constant One
 7531 
 7532 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7533 %{
 7534   match(Set dst con);
 7535 
 7536   ins_cost(INSN_COST);
 7537   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7538 
 7539   ins_encode(aarch64_enc_mov_p1(dst, con));
 7540 
 7541   ins_pipe(ialu_imm);
 7542 %}
 7543 
 7544 // Load Byte Map Base Constant
 7545 
 7546 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7547 %{
 7548   match(Set dst con);
 7549 
 7550   ins_cost(INSN_COST);
 7551   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7552 
 7553   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7554 
 7555   ins_pipe(ialu_imm);
 7556 %}
 7557 
 7558 // Load Narrow Pointer Constant
 7559 
 7560 instruct loadConN(iRegNNoSp dst, immN con)
 7561 %{
 7562   match(Set dst con);
 7563 
 7564   ins_cost(INSN_COST * 4);
 7565   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7566 
 7567   ins_encode(aarch64_enc_mov_n(dst, con));
 7568 
 7569   ins_pipe(ialu_imm);
 7570 %}
 7571 
 7572 // Load Narrow Null Pointer Constant
 7573 
 7574 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7575 %{
 7576   match(Set dst con);
 7577 
 7578   ins_cost(INSN_COST);
 7579   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7580 
 7581   ins_encode(aarch64_enc_mov_n0(dst, con));
 7582 
 7583   ins_pipe(ialu_imm);
 7584 %}
 7585 
 7586 // Load Narrow Klass Constant
 7587 
 7588 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7589 %{
 7590   match(Set dst con);
 7591 
 7592   ins_cost(INSN_COST);
 7593   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7594 
 7595   ins_encode(aarch64_enc_mov_nk(dst, con));
 7596 
 7597   ins_pipe(ialu_imm);
 7598 %}
 7599 
 7600 // Load Packed Float Constant
 7601 
 7602 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7603   match(Set dst con);
 7604   ins_cost(INSN_COST * 4);
 7605   format %{ "fmovs  $dst, $con"%}
 7606   ins_encode %{
 7607     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7608   %}
 7609 
 7610   ins_pipe(fp_imm_s);
 7611 %}
 7612 
 7613 // Load Float Constant
 7614 
 7615 instruct loadConF(vRegF dst, immF con) %{
 7616   match(Set dst con);
 7617 
 7618   ins_cost(INSN_COST * 4);
 7619 
 7620   format %{
 7621     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7622   %}
 7623 
 7624   ins_encode %{
 7625     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7626   %}
 7627 
 7628   ins_pipe(fp_load_constant_s);
 7629 %}
 7630 
 7631 // Load Packed Double Constant
 7632 
 7633 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7634   match(Set dst con);
 7635   ins_cost(INSN_COST);
 7636   format %{ "fmovd  $dst, $con"%}
 7637   ins_encode %{
 7638     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7639   %}
 7640 
 7641   ins_pipe(fp_imm_d);
 7642 %}
 7643 
 7644 // Load Double Constant
 7645 
 7646 instruct loadConD(vRegD dst, immD con) %{
 7647   match(Set dst con);
 7648 
 7649   ins_cost(INSN_COST * 5);
 7650   format %{
 7651     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7652   %}
 7653 
 7654   ins_encode %{
 7655     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7656   %}
 7657 
 7658   ins_pipe(fp_load_constant_d);
 7659 %}
 7660 
 7661 // Store Instructions
 7662 
 7663 // Store CMS card-mark Immediate
 7664 instruct storeimmCM0(immI0 zero, memory1 mem)
 7665 %{
 7666   match(Set mem (StoreCM mem zero));
 7667 
 7668   ins_cost(INSN_COST);
 7669   format %{ "storestore (elided)\n\t"
 7670             "strb zr, $mem\t# byte" %}
 7671 
 7672   ins_encode(aarch64_enc_strb0(mem));
 7673 
 7674   ins_pipe(istore_mem);
 7675 %}
 7676 
 7677 // Store CMS card-mark Immediate with intervening StoreStore
 7678 // needed when using CMS with no conditional card marking
 7679 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7680 %{
 7681   match(Set mem (StoreCM mem zero));
 7682 
 7683   ins_cost(INSN_COST * 2);
 7684   format %{ "storestore\n\t"
 7685             "dmb ishst"
 7686             "\n\tstrb zr, $mem\t# byte" %}
 7687 
 7688   ins_encode(aarch64_enc_strb0_ordered(mem));
 7689 
 7690   ins_pipe(istore_mem);
 7691 %}
 7692 
 7693 // Store Byte
 7694 instruct storeB(iRegIorL2I src, memory1 mem)
 7695 %{
 7696   match(Set mem (StoreB mem src));
 7697   predicate(!needs_releasing_store(n));
 7698 
 7699   ins_cost(INSN_COST);
 7700   format %{ "strb  $src, $mem\t# byte" %}
 7701 
 7702   ins_encode(aarch64_enc_strb(src, mem));
 7703 
 7704   ins_pipe(istore_reg_mem);
 7705 %}
 7706 
 7707 
 7708 instruct storeimmB0(immI0 zero, memory1 mem)
 7709 %{
 7710   match(Set mem (StoreB mem zero));
 7711   predicate(!needs_releasing_store(n));
 7712 
 7713   ins_cost(INSN_COST);
 7714   format %{ "strb rscractch2, $mem\t# byte" %}
 7715 
 7716   ins_encode(aarch64_enc_strb0(mem));
 7717 
 7718   ins_pipe(istore_mem);
 7719 %}
 7720 
 7721 // Store Char/Short
 7722 instruct storeC(iRegIorL2I src, memory2 mem)
 7723 %{
 7724   match(Set mem (StoreC mem src));
 7725   predicate(!needs_releasing_store(n));
 7726 
 7727   ins_cost(INSN_COST);
 7728   format %{ "strh  $src, $mem\t# short" %}
 7729 
 7730   ins_encode(aarch64_enc_strh(src, mem));
 7731 
 7732   ins_pipe(istore_reg_mem);
 7733 %}
 7734 
 7735 instruct storeimmC0(immI0 zero, memory2 mem)
 7736 %{
 7737   match(Set mem (StoreC mem zero));
 7738   predicate(!needs_releasing_store(n));
 7739 
 7740   ins_cost(INSN_COST);
 7741   format %{ "strh  zr, $mem\t# short" %}
 7742 
 7743   ins_encode(aarch64_enc_strh0(mem));
 7744 
 7745   ins_pipe(istore_mem);
 7746 %}
 7747 
 7748 // Store Integer
 7749 
 7750 instruct storeI(iRegIorL2I src, memory4 mem)
 7751 %{
 7752   match(Set mem(StoreI mem src));
 7753   predicate(!needs_releasing_store(n));
 7754 
 7755   ins_cost(INSN_COST);
 7756   format %{ "strw  $src, $mem\t# int" %}
 7757 
 7758   ins_encode(aarch64_enc_strw(src, mem));
 7759 
 7760   ins_pipe(istore_reg_mem);
 7761 %}
 7762 
 7763 instruct storeimmI0(immI0 zero, memory4 mem)
 7764 %{
 7765   match(Set mem(StoreI mem zero));
 7766   predicate(!needs_releasing_store(n));
 7767 
 7768   ins_cost(INSN_COST);
 7769   format %{ "strw  zr, $mem\t# int" %}
 7770 
 7771   ins_encode(aarch64_enc_strw0(mem));
 7772 
 7773   ins_pipe(istore_mem);
 7774 %}
 7775 
 7776 // Store Long (64 bit signed)
 7777 instruct storeL(iRegL src, memory8 mem)
 7778 %{
 7779   match(Set mem (StoreL mem src));
 7780   predicate(!needs_releasing_store(n));
 7781 
 7782   ins_cost(INSN_COST);
 7783   format %{ "str  $src, $mem\t# int" %}
 7784 
 7785   ins_encode(aarch64_enc_str(src, mem));
 7786 
 7787   ins_pipe(istore_reg_mem);
 7788 %}
 7789 
 7790 // Store Long (64 bit signed)
 7791 instruct storeimmL0(immL0 zero, memory8 mem)
 7792 %{
 7793   match(Set mem (StoreL mem zero));
 7794   predicate(!needs_releasing_store(n));
 7795 
 7796   ins_cost(INSN_COST);
 7797   format %{ "str  zr, $mem\t# int" %}
 7798 
 7799   ins_encode(aarch64_enc_str0(mem));
 7800 
 7801   ins_pipe(istore_mem);
 7802 %}
 7803 
 7804 // Store Pointer
 7805 instruct storeP(iRegP src, memory8 mem)
 7806 %{
 7807   match(Set mem (StoreP mem src));
 7808   predicate(!needs_releasing_store(n));
 7809 
 7810   ins_cost(INSN_COST);
 7811   format %{ "str  $src, $mem\t# ptr" %}
 7812 
 7813   ins_encode(aarch64_enc_str(src, mem));
 7814 
 7815   ins_pipe(istore_reg_mem);
 7816 %}
 7817 
 7818 // Store Pointer
 7819 instruct storeimmP0(immP0 zero, memory8 mem)
 7820 %{
 7821   match(Set mem (StoreP mem zero));
 7822   predicate(!needs_releasing_store(n));
 7823 
 7824   ins_cost(INSN_COST);
 7825   format %{ "str zr, $mem\t# ptr" %}
 7826 
 7827   ins_encode(aarch64_enc_str0(mem));
 7828 
 7829   ins_pipe(istore_mem);
 7830 %}
 7831 
 7832 // Store Compressed Pointer
 7833 instruct storeN(iRegN src, memory4 mem)
 7834 %{
 7835   match(Set mem (StoreN mem src));
 7836   predicate(!needs_releasing_store(n));
 7837 
 7838   ins_cost(INSN_COST);
 7839   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7840 
 7841   ins_encode(aarch64_enc_strw(src, mem));
 7842 
 7843   ins_pipe(istore_reg_mem);
 7844 %}
 7845 
 7846 instruct storeImmN0(immN0 zero, memory4 mem)
 7847 %{
 7848   match(Set mem (StoreN mem zero));
 7849   predicate(!needs_releasing_store(n));
 7850 
 7851   ins_cost(INSN_COST);
 7852   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7853 
 7854   ins_encode(aarch64_enc_strw0(mem));
 7855 
 7856   ins_pipe(istore_mem);
 7857 %}
 7858 
 7859 // Store Float
 7860 instruct storeF(vRegF src, memory4 mem)
 7861 %{
 7862   match(Set mem (StoreF mem src));
 7863   predicate(!needs_releasing_store(n));
 7864 
 7865   ins_cost(INSN_COST);
 7866   format %{ "strs  $src, $mem\t# float" %}
 7867 
 7868   ins_encode( aarch64_enc_strs(src, mem) );
 7869 
 7870   ins_pipe(pipe_class_memory);
 7871 %}
 7872 
 7873 // TODO
 7874 // implement storeImmF0 and storeFImmPacked
 7875 
 7876 // Store Double
 7877 instruct storeD(vRegD src, memory8 mem)
 7878 %{
 7879   match(Set mem (StoreD mem src));
 7880   predicate(!needs_releasing_store(n));
 7881 
 7882   ins_cost(INSN_COST);
 7883   format %{ "strd  $src, $mem\t# double" %}
 7884 
 7885   ins_encode( aarch64_enc_strd(src, mem) );
 7886 
 7887   ins_pipe(pipe_class_memory);
 7888 %}
 7889 
 7890 // Store Compressed Klass Pointer
 7891 instruct storeNKlass(iRegN src, memory4 mem)
 7892 %{
 7893   predicate(!needs_releasing_store(n));
 7894   match(Set mem (StoreNKlass mem src));
 7895 
 7896   ins_cost(INSN_COST);
 7897   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7898 
 7899   ins_encode(aarch64_enc_strw(src, mem));
 7900 
 7901   ins_pipe(istore_reg_mem);
 7902 %}
 7903 
 7904 // TODO
 7905 // implement storeImmD0 and storeDImmPacked
 7906 
 7907 // prefetch instructions
 7908 // Must be safe to execute with invalid address (cannot fault).
 7909 
 7910 instruct prefetchalloc( memory8 mem ) %{
 7911   match(PrefetchAllocation mem);
 7912 
 7913   ins_cost(INSN_COST);
 7914   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7915 
 7916   ins_encode( aarch64_enc_prefetchw(mem) );
 7917 
 7918   ins_pipe(iload_prefetch);
 7919 %}
 7920 
 7921 //  ---------------- volatile loads and stores ----------------
 7922 
 7923 // Load Byte (8 bit signed)
 7924 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7925 %{
 7926   match(Set dst (LoadB mem));
 7927 
 7928   ins_cost(VOLATILE_REF_COST);
 7929   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7930 
 7931   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7932 
 7933   ins_pipe(pipe_serial);
 7934 %}
 7935 
 7936 // Load Byte (8 bit signed) into long
 7937 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7938 %{
 7939   match(Set dst (ConvI2L (LoadB mem)));
 7940 
 7941   ins_cost(VOLATILE_REF_COST);
 7942   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7943 
 7944   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7945 
 7946   ins_pipe(pipe_serial);
 7947 %}
 7948 
 7949 // Load Byte (8 bit unsigned)
 7950 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7951 %{
 7952   match(Set dst (LoadUB mem));
 7953 
 7954   ins_cost(VOLATILE_REF_COST);
 7955   format %{ "ldarb  $dst, $mem\t# byte" %}
 7956 
 7957   ins_encode(aarch64_enc_ldarb(dst, mem));
 7958 
 7959   ins_pipe(pipe_serial);
 7960 %}
 7961 
 7962 // Load Byte (8 bit unsigned) into long
 7963 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7964 %{
 7965   match(Set dst (ConvI2L (LoadUB mem)));
 7966 
 7967   ins_cost(VOLATILE_REF_COST);
 7968   format %{ "ldarb  $dst, $mem\t# byte" %}
 7969 
 7970   ins_encode(aarch64_enc_ldarb(dst, mem));
 7971 
 7972   ins_pipe(pipe_serial);
 7973 %}
 7974 
 7975 // Load Short (16 bit signed)
 7976 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7977 %{
 7978   match(Set dst (LoadS mem));
 7979 
 7980   ins_cost(VOLATILE_REF_COST);
 7981   format %{ "ldarshw  $dst, $mem\t# short" %}
 7982 
 7983   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7984 
 7985   ins_pipe(pipe_serial);
 7986 %}
 7987 
 7988 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7989 %{
 7990   match(Set dst (LoadUS mem));
 7991 
 7992   ins_cost(VOLATILE_REF_COST);
 7993   format %{ "ldarhw  $dst, $mem\t# short" %}
 7994 
 7995   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7996 
 7997   ins_pipe(pipe_serial);
 7998 %}
 7999 
 8000 // Load Short/Char (16 bit unsigned) into long
 8001 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8002 %{
 8003   match(Set dst (ConvI2L (LoadUS mem)));
 8004 
 8005   ins_cost(VOLATILE_REF_COST);
 8006   format %{ "ldarh  $dst, $mem\t# short" %}
 8007 
 8008   ins_encode(aarch64_enc_ldarh(dst, mem));
 8009 
 8010   ins_pipe(pipe_serial);
 8011 %}
 8012 
 8013 // Load Short/Char (16 bit signed) into long
 8014 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8015 %{
 8016   match(Set dst (ConvI2L (LoadS mem)));
 8017 
 8018   ins_cost(VOLATILE_REF_COST);
 8019   format %{ "ldarh  $dst, $mem\t# short" %}
 8020 
 8021   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8022 
 8023   ins_pipe(pipe_serial);
 8024 %}
 8025 
 8026 // Load Integer (32 bit signed)
 8027 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8028 %{
 8029   match(Set dst (LoadI mem));
 8030 
 8031   ins_cost(VOLATILE_REF_COST);
 8032   format %{ "ldarw  $dst, $mem\t# int" %}
 8033 
 8034   ins_encode(aarch64_enc_ldarw(dst, mem));
 8035 
 8036   ins_pipe(pipe_serial);
 8037 %}
 8038 
 8039 // Load Integer (32 bit unsigned) into long
 8040 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8041 %{
 8042   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8043 
 8044   ins_cost(VOLATILE_REF_COST);
 8045   format %{ "ldarw  $dst, $mem\t# int" %}
 8046 
 8047   ins_encode(aarch64_enc_ldarw(dst, mem));
 8048 
 8049   ins_pipe(pipe_serial);
 8050 %}
 8051 
 8052 // Load Long (64 bit signed)
 8053 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8054 %{
 8055   match(Set dst (LoadL mem));
 8056 
 8057   ins_cost(VOLATILE_REF_COST);
 8058   format %{ "ldar  $dst, $mem\t# int" %}
 8059 
 8060   ins_encode(aarch64_enc_ldar(dst, mem));
 8061 
 8062   ins_pipe(pipe_serial);
 8063 %}
 8064 
 8065 // Load Pointer
 8066 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8067 %{
 8068   match(Set dst (LoadP mem));
 8069   predicate(n->as_Load()->barrier_data() == 0);
 8070 
 8071   ins_cost(VOLATILE_REF_COST);
 8072   format %{ "ldar  $dst, $mem\t# ptr" %}
 8073 
 8074   ins_encode(aarch64_enc_ldar(dst, mem));
 8075 
 8076   ins_pipe(pipe_serial);
 8077 %}
 8078 
 8079 // Load Compressed Pointer
 8080 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8081 %{
 8082   match(Set dst (LoadN mem));
 8083 
 8084   ins_cost(VOLATILE_REF_COST);
 8085   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8086 
 8087   ins_encode(aarch64_enc_ldarw(dst, mem));
 8088 
 8089   ins_pipe(pipe_serial);
 8090 %}
 8091 
 8092 // Load Float
 8093 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8094 %{
 8095   match(Set dst (LoadF mem));
 8096 
 8097   ins_cost(VOLATILE_REF_COST);
 8098   format %{ "ldars  $dst, $mem\t# float" %}
 8099 
 8100   ins_encode( aarch64_enc_fldars(dst, mem) );
 8101 
 8102   ins_pipe(pipe_serial);
 8103 %}
 8104 
 8105 // Load Double
 8106 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8107 %{
 8108   match(Set dst (LoadD mem));
 8109 
 8110   ins_cost(VOLATILE_REF_COST);
 8111   format %{ "ldard  $dst, $mem\t# double" %}
 8112 
 8113   ins_encode( aarch64_enc_fldard(dst, mem) );
 8114 
 8115   ins_pipe(pipe_serial);
 8116 %}
 8117 
 8118 // Store Byte
 8119 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8120 %{
 8121   match(Set mem (StoreB mem src));
 8122 
 8123   ins_cost(VOLATILE_REF_COST);
 8124   format %{ "stlrb  $src, $mem\t# byte" %}
 8125 
 8126   ins_encode(aarch64_enc_stlrb(src, mem));
 8127 
 8128   ins_pipe(pipe_class_memory);
 8129 %}
 8130 
 8131 // Store Char/Short
 8132 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8133 %{
 8134   match(Set mem (StoreC mem src));
 8135 
 8136   ins_cost(VOLATILE_REF_COST);
 8137   format %{ "stlrh  $src, $mem\t# short" %}
 8138 
 8139   ins_encode(aarch64_enc_stlrh(src, mem));
 8140 
 8141   ins_pipe(pipe_class_memory);
 8142 %}
 8143 
 8144 // Store Integer
 8145 
 8146 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8147 %{
 8148   match(Set mem(StoreI mem src));
 8149 
 8150   ins_cost(VOLATILE_REF_COST);
 8151   format %{ "stlrw  $src, $mem\t# int" %}
 8152 
 8153   ins_encode(aarch64_enc_stlrw(src, mem));
 8154 
 8155   ins_pipe(pipe_class_memory);
 8156 %}
 8157 
 8158 // Store Long (64 bit signed)
 8159 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8160 %{
 8161   match(Set mem (StoreL mem src));
 8162 
 8163   ins_cost(VOLATILE_REF_COST);
 8164   format %{ "stlr  $src, $mem\t# int" %}
 8165 
 8166   ins_encode(aarch64_enc_stlr(src, mem));
 8167 
 8168   ins_pipe(pipe_class_memory);
 8169 %}
 8170 
 8171 // Store Pointer
 8172 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8173 %{
 8174   match(Set mem (StoreP mem src));
 8175 
 8176   ins_cost(VOLATILE_REF_COST);
 8177   format %{ "stlr  $src, $mem\t# ptr" %}
 8178 
 8179   ins_encode(aarch64_enc_stlr(src, mem));
 8180 
 8181   ins_pipe(pipe_class_memory);
 8182 %}
 8183 
 8184 // Store Compressed Pointer
 8185 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8186 %{
 8187   match(Set mem (StoreN mem src));
 8188 
 8189   ins_cost(VOLATILE_REF_COST);
 8190   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8191 
 8192   ins_encode(aarch64_enc_stlrw(src, mem));
 8193 
 8194   ins_pipe(pipe_class_memory);
 8195 %}
 8196 
 8197 // Store Float
 8198 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8199 %{
 8200   match(Set mem (StoreF mem src));
 8201 
 8202   ins_cost(VOLATILE_REF_COST);
 8203   format %{ "stlrs  $src, $mem\t# float" %}
 8204 
 8205   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8206 
 8207   ins_pipe(pipe_class_memory);
 8208 %}
 8209 
 8210 // TODO
 8211 // implement storeImmF0 and storeFImmPacked
 8212 
 8213 // Store Double
 8214 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8215 %{
 8216   match(Set mem (StoreD mem src));
 8217 
 8218   ins_cost(VOLATILE_REF_COST);
 8219   format %{ "stlrd  $src, $mem\t# double" %}
 8220 
 8221   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8222 
 8223   ins_pipe(pipe_class_memory);
 8224 %}
 8225 
 8226 //  ---------------- end of volatile loads and stores ----------------
 8227 
 8228 instruct cacheWB(indirect addr)
 8229 %{
 8230   predicate(VM_Version::supports_data_cache_line_flush());
 8231   match(CacheWB addr);
 8232 
 8233   ins_cost(100);
 8234   format %{"cache wb $addr" %}
 8235   ins_encode %{
 8236     assert($addr->index_position() < 0, "should be");
 8237     assert($addr$$disp == 0, "should be");
 8238     __ cache_wb(Address($addr$$base$$Register, 0));
 8239   %}
 8240   ins_pipe(pipe_slow); // XXX
 8241 %}
 8242 
 8243 instruct cacheWBPreSync()
 8244 %{
 8245   predicate(VM_Version::supports_data_cache_line_flush());
 8246   match(CacheWBPreSync);
 8247 
 8248   ins_cost(100);
 8249   format %{"cache wb presync" %}
 8250   ins_encode %{
 8251     __ cache_wbsync(true);
 8252   %}
 8253   ins_pipe(pipe_slow); // XXX
 8254 %}
 8255 
 8256 instruct cacheWBPostSync()
 8257 %{
 8258   predicate(VM_Version::supports_data_cache_line_flush());
 8259   match(CacheWBPostSync);
 8260 
 8261   ins_cost(100);
 8262   format %{"cache wb postsync" %}
 8263   ins_encode %{
 8264     __ cache_wbsync(false);
 8265   %}
 8266   ins_pipe(pipe_slow); // XXX
 8267 %}
 8268 
 8269 // ============================================================================
 8270 // BSWAP Instructions
 8271 
 8272 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8273   match(Set dst (ReverseBytesI src));
 8274 
 8275   ins_cost(INSN_COST);
 8276   format %{ "revw  $dst, $src" %}
 8277 
 8278   ins_encode %{
 8279     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8280   %}
 8281 
 8282   ins_pipe(ialu_reg);
 8283 %}
 8284 
 8285 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8286   match(Set dst (ReverseBytesL src));
 8287 
 8288   ins_cost(INSN_COST);
 8289   format %{ "rev  $dst, $src" %}
 8290 
 8291   ins_encode %{
 8292     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8293   %}
 8294 
 8295   ins_pipe(ialu_reg);
 8296 %}
 8297 
 8298 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8299   match(Set dst (ReverseBytesUS src));
 8300 
 8301   ins_cost(INSN_COST);
 8302   format %{ "rev16w  $dst, $src" %}
 8303 
 8304   ins_encode %{
 8305     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8306   %}
 8307 
 8308   ins_pipe(ialu_reg);
 8309 %}
 8310 
 8311 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8312   match(Set dst (ReverseBytesS src));
 8313 
 8314   ins_cost(INSN_COST);
 8315   format %{ "rev16w  $dst, $src\n\t"
 8316             "sbfmw $dst, $dst, #0, #15" %}
 8317 
 8318   ins_encode %{
 8319     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8320     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8321   %}
 8322 
 8323   ins_pipe(ialu_reg);
 8324 %}
 8325 
 8326 // ============================================================================
 8327 // Zero Count Instructions
 8328 
 8329 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8330   match(Set dst (CountLeadingZerosI src));
 8331 
 8332   ins_cost(INSN_COST);
 8333   format %{ "clzw  $dst, $src" %}
 8334   ins_encode %{
 8335     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8336   %}
 8337 
 8338   ins_pipe(ialu_reg);
 8339 %}
 8340 
 8341 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8342   match(Set dst (CountLeadingZerosL src));
 8343 
 8344   ins_cost(INSN_COST);
 8345   format %{ "clz   $dst, $src" %}
 8346   ins_encode %{
 8347     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8348   %}
 8349 
 8350   ins_pipe(ialu_reg);
 8351 %}
 8352 
 8353 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8354   match(Set dst (CountTrailingZerosI src));
 8355 
 8356   ins_cost(INSN_COST * 2);
 8357   format %{ "rbitw  $dst, $src\n\t"
 8358             "clzw   $dst, $dst" %}
 8359   ins_encode %{
 8360     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8361     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8362   %}
 8363 
 8364   ins_pipe(ialu_reg);
 8365 %}
 8366 
 8367 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8368   match(Set dst (CountTrailingZerosL src));
 8369 
 8370   ins_cost(INSN_COST * 2);
 8371   format %{ "rbit   $dst, $src\n\t"
 8372             "clz    $dst, $dst" %}
 8373   ins_encode %{
 8374     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8375     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8376   %}
 8377 
 8378   ins_pipe(ialu_reg);
 8379 %}
 8380 
 8381 //---------- Population Count Instructions -------------------------------------
 8382 //
 8383 
 8384 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8385   predicate(UsePopCountInstruction);
 8386   match(Set dst (PopCountI src));
 8387   effect(TEMP tmp);
 8388   ins_cost(INSN_COST * 13);
 8389 
 8390   format %{ "movw   $src, $src\n\t"
 8391             "mov    $tmp, $src\t# vector (1D)\n\t"
 8392             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8393             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8394             "mov    $dst, $tmp\t# vector (1D)" %}
 8395   ins_encode %{
 8396     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8397     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8398     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8399     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8400     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8401   %}
 8402 
 8403   ins_pipe(pipe_class_default);
 8404 %}
 8405 
 8406 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8407   predicate(UsePopCountInstruction);
 8408   match(Set dst (PopCountI (LoadI mem)));
 8409   effect(TEMP tmp);
 8410   ins_cost(INSN_COST * 13);
 8411 
 8412   format %{ "ldrs   $tmp, $mem\n\t"
 8413             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8414             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8415             "mov    $dst, $tmp\t# vector (1D)" %}
 8416   ins_encode %{
 8417     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8418     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8419               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8420     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8421     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8422     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8423   %}
 8424 
 8425   ins_pipe(pipe_class_default);
 8426 %}
 8427 
 8428 // Note: Long.bitCount(long) returns an int.
 8429 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8430   predicate(UsePopCountInstruction);
 8431   match(Set dst (PopCountL src));
 8432   effect(TEMP tmp);
 8433   ins_cost(INSN_COST * 13);
 8434 
 8435   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8436             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8437             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8438             "mov    $dst, $tmp\t# vector (1D)" %}
 8439   ins_encode %{
 8440     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8441     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8442     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8443     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8444   %}
 8445 
 8446   ins_pipe(pipe_class_default);
 8447 %}
 8448 
 8449 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8450   predicate(UsePopCountInstruction);
 8451   match(Set dst (PopCountL (LoadL mem)));
 8452   effect(TEMP tmp);
 8453   ins_cost(INSN_COST * 13);
 8454 
 8455   format %{ "ldrd   $tmp, $mem\n\t"
 8456             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8457             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8458             "mov    $dst, $tmp\t# vector (1D)" %}
 8459   ins_encode %{
 8460     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8461     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8462               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8463     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8464     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8465     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8466   %}
 8467 
 8468   ins_pipe(pipe_class_default);
 8469 %}
 8470 
 8471 // ============================================================================
 8472 // MemBar Instruction
 8473 
 8474 instruct load_fence() %{
 8475   match(LoadFence);
 8476   ins_cost(VOLATILE_REF_COST);
 8477 
 8478   format %{ "load_fence" %}
 8479 
 8480   ins_encode %{
 8481     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8482   %}
 8483   ins_pipe(pipe_serial);
 8484 %}
 8485 
 8486 instruct unnecessary_membar_acquire() %{
 8487   predicate(unnecessary_acquire(n));
 8488   match(MemBarAcquire);
 8489   ins_cost(0);
 8490 
 8491   format %{ "membar_acquire (elided)" %}
 8492 
 8493   ins_encode %{
 8494     __ block_comment("membar_acquire (elided)");
 8495   %}
 8496 
 8497   ins_pipe(pipe_class_empty);
 8498 %}
 8499 
 8500 instruct membar_acquire() %{
 8501   match(MemBarAcquire);
 8502   ins_cost(VOLATILE_REF_COST);
 8503 
 8504   format %{ "membar_acquire\n\t"
 8505             "dmb ish" %}
 8506 
 8507   ins_encode %{
 8508     __ block_comment("membar_acquire");
 8509     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8510   %}
 8511 
 8512   ins_pipe(pipe_serial);
 8513 %}
 8514 
 8515 
 8516 instruct membar_acquire_lock() %{
 8517   match(MemBarAcquireLock);
 8518   ins_cost(VOLATILE_REF_COST);
 8519 
 8520   format %{ "membar_acquire_lock (elided)" %}
 8521 
 8522   ins_encode %{
 8523     __ block_comment("membar_acquire_lock (elided)");
 8524   %}
 8525 
 8526   ins_pipe(pipe_serial);
 8527 %}
 8528 
 8529 instruct store_fence() %{
 8530   match(StoreFence);
 8531   ins_cost(VOLATILE_REF_COST);
 8532 
 8533   format %{ "store_fence" %}
 8534 
 8535   ins_encode %{
 8536     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8537   %}
 8538   ins_pipe(pipe_serial);
 8539 %}
 8540 
 8541 instruct unnecessary_membar_release() %{
 8542   predicate(unnecessary_release(n));
 8543   match(MemBarRelease);
 8544   ins_cost(0);
 8545 
 8546   format %{ "membar_release (elided)" %}
 8547 
 8548   ins_encode %{
 8549     __ block_comment("membar_release (elided)");
 8550   %}
 8551   ins_pipe(pipe_serial);
 8552 %}
 8553 
 8554 instruct membar_release() %{
 8555   match(MemBarRelease);
 8556   ins_cost(VOLATILE_REF_COST);
 8557 
 8558   format %{ "membar_release\n\t"
 8559             "dmb ish" %}
 8560 
 8561   ins_encode %{
 8562     __ block_comment("membar_release");
 8563     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8564   %}
 8565   ins_pipe(pipe_serial);
 8566 %}
 8567 
 8568 instruct membar_storestore() %{
 8569   match(MemBarStoreStore);
 8570   ins_cost(VOLATILE_REF_COST);
 8571 
 8572   format %{ "MEMBAR-store-store" %}
 8573 
 8574   ins_encode %{
 8575     __ membar(Assembler::StoreStore);
 8576   %}
 8577   ins_pipe(pipe_serial);
 8578 %}
 8579 
 8580 instruct membar_release_lock() %{
 8581   match(MemBarReleaseLock);
 8582   ins_cost(VOLATILE_REF_COST);
 8583 
 8584   format %{ "membar_release_lock (elided)" %}
 8585 
 8586   ins_encode %{
 8587     __ block_comment("membar_release_lock (elided)");
 8588   %}
 8589 
 8590   ins_pipe(pipe_serial);
 8591 %}
 8592 
 8593 instruct unnecessary_membar_volatile() %{
 8594   predicate(unnecessary_volatile(n));
 8595   match(MemBarVolatile);
 8596   ins_cost(0);
 8597 
 8598   format %{ "membar_volatile (elided)" %}
 8599 
 8600   ins_encode %{
 8601     __ block_comment("membar_volatile (elided)");
 8602   %}
 8603 
 8604   ins_pipe(pipe_serial);
 8605 %}
 8606 
 8607 instruct membar_volatile() %{
 8608   match(MemBarVolatile);
 8609   ins_cost(VOLATILE_REF_COST*100);
 8610 
 8611   format %{ "membar_volatile\n\t"
 8612              "dmb ish"%}
 8613 
 8614   ins_encode %{
 8615     __ block_comment("membar_volatile");
 8616     __ membar(Assembler::StoreLoad);
 8617   %}
 8618 
 8619   ins_pipe(pipe_serial);
 8620 %}
 8621 
 8622 // ============================================================================
 8623 // Cast/Convert Instructions
 8624 
 8625 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8626   match(Set dst (CastX2P src));
 8627 
 8628   ins_cost(INSN_COST);
 8629   format %{ "mov $dst, $src\t# long -> ptr" %}
 8630 
 8631   ins_encode %{
 8632     if ($dst$$reg != $src$$reg) {
 8633       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8634     }
 8635   %}
 8636 
 8637   ins_pipe(ialu_reg);
 8638 %}
 8639 
 8640 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8641   match(Set dst (CastP2X src));
 8642 
 8643   ins_cost(INSN_COST);
 8644   format %{ "mov $dst, $src\t# ptr -> long" %}
 8645 
 8646   ins_encode %{
 8647     if ($dst$$reg != $src$$reg) {
 8648       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8649     }
 8650   %}
 8651 
 8652   ins_pipe(ialu_reg);
 8653 %}
 8654 
 8655 // Convert oop into int for vectors alignment masking
 8656 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8657   match(Set dst (ConvL2I (CastP2X src)));
 8658 
 8659   ins_cost(INSN_COST);
 8660   format %{ "movw $dst, $src\t# ptr -> int" %}
 8661   ins_encode %{
 8662     __ movw($dst$$Register, $src$$Register);
 8663   %}
 8664 
 8665   ins_pipe(ialu_reg);
 8666 %}
 8667 
 8668 // Convert compressed oop into int for vectors alignment masking
 8669 // in case of 32bit oops (heap < 4Gb).
 8670 instruct convN2I(iRegINoSp dst, iRegN src)
 8671 %{
 8672   predicate(CompressedOops::shift() == 0);
 8673   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8674 
 8675   ins_cost(INSN_COST);
 8676   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8677   ins_encode %{
 8678     __ movw($dst$$Register, $src$$Register);
 8679   %}
 8680 
 8681   ins_pipe(ialu_reg);
 8682 %}
 8683 
 8684 
 8685 // Convert oop pointer into compressed form
 8686 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8687   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8688   match(Set dst (EncodeP src));
 8689   effect(KILL cr);
 8690   ins_cost(INSN_COST * 3);
 8691   format %{ "encode_heap_oop $dst, $src" %}
 8692   ins_encode %{
 8693     Register s = $src$$Register;
 8694     Register d = $dst$$Register;
 8695     __ encode_heap_oop(d, s);
 8696   %}
 8697   ins_pipe(ialu_reg);
 8698 %}
 8699 
 8700 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8701   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8702   match(Set dst (EncodeP src));
 8703   ins_cost(INSN_COST * 3);
 8704   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8705   ins_encode %{
 8706     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8707   %}
 8708   ins_pipe(ialu_reg);
 8709 %}
 8710 
 8711 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8712   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8713             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8714   match(Set dst (DecodeN src));
 8715   ins_cost(INSN_COST * 3);
 8716   format %{ "decode_heap_oop $dst, $src" %}
 8717   ins_encode %{
 8718     Register s = $src$$Register;
 8719     Register d = $dst$$Register;
 8720     __ decode_heap_oop(d, s);
 8721   %}
 8722   ins_pipe(ialu_reg);
 8723 %}
 8724 
 8725 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8726   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8727             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8728   match(Set dst (DecodeN src));
 8729   ins_cost(INSN_COST * 3);
 8730   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8731   ins_encode %{
 8732     Register s = $src$$Register;
 8733     Register d = $dst$$Register;
 8734     __ decode_heap_oop_not_null(d, s);
 8735   %}
 8736   ins_pipe(ialu_reg);
 8737 %}
 8738 
 8739 // n.b. AArch64 implementations of encode_klass_not_null and
 8740 // decode_klass_not_null do not modify the flags register so, unlike
 8741 // Intel, we don't kill CR as a side effect here
 8742 
 8743 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8744   match(Set dst (EncodePKlass src));
 8745 
 8746   ins_cost(INSN_COST * 3);
 8747   format %{ "encode_klass_not_null $dst,$src" %}
 8748 
 8749   ins_encode %{
 8750     Register src_reg = as_Register($src$$reg);
 8751     Register dst_reg = as_Register($dst$$reg);
 8752     __ encode_klass_not_null(dst_reg, src_reg);
 8753   %}
 8754 
 8755    ins_pipe(ialu_reg);
 8756 %}
 8757 
 8758 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8759   match(Set dst (DecodeNKlass src));
 8760 
 8761   ins_cost(INSN_COST * 3);
 8762   format %{ "decode_klass_not_null $dst,$src" %}
 8763 
 8764   ins_encode %{
 8765     Register src_reg = as_Register($src$$reg);
 8766     Register dst_reg = as_Register($dst$$reg);
 8767     if (dst_reg != src_reg) {
 8768       __ decode_klass_not_null(dst_reg, src_reg);
 8769     } else {
 8770       __ decode_klass_not_null(dst_reg);
 8771     }
 8772   %}
 8773 
 8774    ins_pipe(ialu_reg);
 8775 %}
 8776 
 8777 instruct checkCastPP(iRegPNoSp dst)
 8778 %{
 8779   match(Set dst (CheckCastPP dst));
 8780 
 8781   size(0);
 8782   format %{ "# checkcastPP of $dst" %}
 8783   ins_encode(/* empty encoding */);
 8784   ins_pipe(pipe_class_empty);
 8785 %}
 8786 
 8787 instruct castPP(iRegPNoSp dst)
 8788 %{
 8789   match(Set dst (CastPP dst));
 8790 
 8791   size(0);
 8792   format %{ "# castPP of $dst" %}
 8793   ins_encode(/* empty encoding */);
 8794   ins_pipe(pipe_class_empty);
 8795 %}
 8796 
 8797 instruct castII(iRegI dst)
 8798 %{
 8799   match(Set dst (CastII dst));
 8800 
 8801   size(0);
 8802   format %{ "# castII of $dst" %}
 8803   ins_encode(/* empty encoding */);
 8804   ins_cost(0);
 8805   ins_pipe(pipe_class_empty);
 8806 %}
 8807 
 8808 instruct castLL(iRegL dst)
 8809 %{
 8810   match(Set dst (CastLL dst));
 8811 
 8812   size(0);
 8813   format %{ "# castLL of $dst" %}
 8814   ins_encode(/* empty encoding */);
 8815   ins_cost(0);
 8816   ins_pipe(pipe_class_empty);
 8817 %}
 8818 
 8819 instruct castFF(vRegF dst)
 8820 %{
 8821   match(Set dst (CastFF dst));
 8822 
 8823   size(0);
 8824   format %{ "# castFF of $dst" %}
 8825   ins_encode(/* empty encoding */);
 8826   ins_cost(0);
 8827   ins_pipe(pipe_class_empty);
 8828 %}
 8829 
 8830 instruct castDD(vRegD dst)
 8831 %{
 8832   match(Set dst (CastDD dst));
 8833 
 8834   size(0);
 8835   format %{ "# castDD of $dst" %}
 8836   ins_encode(/* empty encoding */);
 8837   ins_cost(0);
 8838   ins_pipe(pipe_class_empty);
 8839 %}
 8840 
 8841 instruct castVVD(vecD dst)
 8842 %{
 8843   match(Set dst (CastVV dst));
 8844 
 8845   size(0);
 8846   format %{ "# castVV of $dst" %}
 8847   ins_encode(/* empty encoding */);
 8848   ins_cost(0);
 8849   ins_pipe(pipe_class_empty);
 8850 %}
 8851 
 8852 instruct castVVX(vecX dst)
 8853 %{
 8854   match(Set dst (CastVV dst));
 8855 
 8856   size(0);
 8857   format %{ "# castVV of $dst" %}
 8858   ins_encode(/* empty encoding */);
 8859   ins_cost(0);
 8860   ins_pipe(pipe_class_empty);
 8861 %}
 8862 
 8863 instruct castVV(vReg dst)
 8864 %{
 8865   match(Set dst (CastVV dst));
 8866 
 8867   size(0);
 8868   format %{ "# castVV of $dst" %}
 8869   ins_encode(/* empty encoding */);
 8870   ins_cost(0);
 8871   ins_pipe(pipe_class_empty);
 8872 %}
 8873 
 8874 // ============================================================================
 8875 // Atomic operation instructions
 8876 //
 8877 // Intel and SPARC both implement Ideal Node LoadPLocked and
 8878 // Store{PIL}Conditional instructions using a normal load for the
 8879 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 8880 //
 8881 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 8882 // pair to lock object allocations from Eden space when not using
 8883 // TLABs.
 8884 //
 8885 // There does not appear to be a Load{IL}Locked Ideal Node and the
 8886 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 8887 // and to use StoreIConditional only for 32-bit and StoreLConditional
 8888 // only for 64-bit.
 8889 //
 8890 // We implement LoadPLocked and StorePLocked instructions using,
 8891 // respectively the AArch64 hw load-exclusive and store-conditional
 8892 // instructions. Whereas we must implement each of
 8893 // Store{IL}Conditional using a CAS which employs a pair of
 8894 // instructions comprising a load-exclusive followed by a
 8895 // store-conditional.
 8896 
 8897 
 8898 // Locked-load (linked load) of the current heap-top
 8899 // used when updating the eden heap top
 8900 // implemented using ldaxr on AArch64
 8901 
 8902 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 8903 %{
 8904   match(Set dst (LoadPLocked mem));
 8905 
 8906   ins_cost(VOLATILE_REF_COST);
 8907 
 8908   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 8909 
 8910   ins_encode(aarch64_enc_ldaxr(dst, mem));
 8911 
 8912   ins_pipe(pipe_serial);
 8913 %}
 8914 
 8915 // Conditional-store of the updated heap-top.
 8916 // Used during allocation of the shared heap.
 8917 // Sets flag (EQ) on success.
 8918 // implemented using stlxr on AArch64.
 8919 
 8920 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 8921 %{
 8922   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 8923 
 8924   ins_cost(VOLATILE_REF_COST);
 8925 
 8926  // TODO
 8927  // do we need to do a store-conditional release or can we just use a
 8928  // plain store-conditional?
 8929 
 8930   format %{
 8931     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 8932     "cmpw rscratch1, zr\t# EQ on successful write"
 8933   %}
 8934 
 8935   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 8936 
 8937   ins_pipe(pipe_serial);
 8938 %}
 8939 
 8940 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 8941 %{
 8942   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 8943 
 8944   ins_cost(VOLATILE_REF_COST);
 8945 
 8946   format %{
 8947     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 8948     "cmpw rscratch1, zr\t# EQ on successful write"
 8949   %}
 8950 
 8951   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 8952 
 8953   ins_pipe(pipe_slow);
 8954 %}
 8955 
 8956 // storeIConditional also has acquire semantics, for no better reason
 8957 // than matching storeLConditional.  At the time of writing this
 8958 // comment storeIConditional was not used anywhere by AArch64.
 8959 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 8960 %{
 8961   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 8962 
 8963   ins_cost(VOLATILE_REF_COST);
 8964 
 8965   format %{
 8966     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 8967     "cmpw rscratch1, zr\t# EQ on successful write"
 8968   %}
 8969 
 8970   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 8971 
 8972   ins_pipe(pipe_slow);
 8973 %}
 8974 
 8975 // standard CompareAndSwapX when we are using barriers
 8976 // these have higher priority than the rules selected by a predicate
 8977 
 8978 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8979 // can't match them
 8980 
 8981 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8982 
 8983   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8984   ins_cost(2 * VOLATILE_REF_COST);
 8985 
 8986   effect(KILL cr);
 8987 
 8988   format %{
 8989     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8990     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8991   %}
 8992 
 8993   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8994             aarch64_enc_cset_eq(res));
 8995 
 8996   ins_pipe(pipe_slow);
 8997 %}
 8998 
 8999 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9000 
 9001   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9002   ins_cost(2 * VOLATILE_REF_COST);
 9003 
 9004   effect(KILL cr);
 9005 
 9006   format %{
 9007     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9008     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9009   %}
 9010 
 9011   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9012             aarch64_enc_cset_eq(res));
 9013 
 9014   ins_pipe(pipe_slow);
 9015 %}
 9016 
 9017 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9018 
 9019   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9020   ins_cost(2 * VOLATILE_REF_COST);
 9021 
 9022   effect(KILL cr);
 9023 
 9024  format %{
 9025     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9026     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9027  %}
 9028 
 9029  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9030             aarch64_enc_cset_eq(res));
 9031 
 9032   ins_pipe(pipe_slow);
 9033 %}
 9034 
 9035 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9036 
 9037   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9038   ins_cost(2 * VOLATILE_REF_COST);
 9039 
 9040   effect(KILL cr);
 9041 
 9042  format %{
 9043     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9044     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9045  %}
 9046 
 9047  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9048             aarch64_enc_cset_eq(res));
 9049 
 9050   ins_pipe(pipe_slow);
 9051 %}
 9052 
 9053 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9054 
 9055   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9056   predicate(n->as_LoadStore()->barrier_data() == 0);
 9057   ins_cost(2 * VOLATILE_REF_COST);
 9058 
 9059   effect(KILL cr);
 9060 
 9061  format %{
 9062     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9063     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9064  %}
 9065 
 9066  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9067             aarch64_enc_cset_eq(res));
 9068 
 9069   ins_pipe(pipe_slow);
 9070 %}
 9071 
 9072 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9073 
 9074   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9075   ins_cost(2 * VOLATILE_REF_COST);
 9076 
 9077   effect(KILL cr);
 9078 
 9079  format %{
 9080     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9081     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9082  %}
 9083 
 9084  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9085             aarch64_enc_cset_eq(res));
 9086 
 9087   ins_pipe(pipe_slow);
 9088 %}
 9089 
 9090 // alternative CompareAndSwapX when we are eliding barriers
 9091 
 9092 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9093 
 9094   predicate(needs_acquiring_load_exclusive(n));
 9095   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9096   ins_cost(VOLATILE_REF_COST);
 9097 
 9098   effect(KILL cr);
 9099 
 9100   format %{
 9101     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9102     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9103   %}
 9104 
 9105   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9106             aarch64_enc_cset_eq(res));
 9107 
 9108   ins_pipe(pipe_slow);
 9109 %}
 9110 
 9111 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9112 
 9113   predicate(needs_acquiring_load_exclusive(n));
 9114   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9115   ins_cost(VOLATILE_REF_COST);
 9116 
 9117   effect(KILL cr);
 9118 
 9119   format %{
 9120     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9121     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9122   %}
 9123 
 9124   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9125             aarch64_enc_cset_eq(res));
 9126 
 9127   ins_pipe(pipe_slow);
 9128 %}
 9129 
 9130 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9131 
 9132   predicate(needs_acquiring_load_exclusive(n));
 9133   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9134   ins_cost(VOLATILE_REF_COST);
 9135 
 9136   effect(KILL cr);
 9137 
 9138  format %{
 9139     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9140     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9141  %}
 9142 
 9143  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9144             aarch64_enc_cset_eq(res));
 9145 
 9146   ins_pipe(pipe_slow);
 9147 %}
 9148 
 9149 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9150 
 9151   predicate(needs_acquiring_load_exclusive(n));
 9152   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9153   ins_cost(VOLATILE_REF_COST);
 9154 
 9155   effect(KILL cr);
 9156 
 9157  format %{
 9158     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9159     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9160  %}
 9161 
 9162  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9163             aarch64_enc_cset_eq(res));
 9164 
 9165   ins_pipe(pipe_slow);
 9166 %}
 9167 
 9168 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9169 
 9170   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9171   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9172   ins_cost(VOLATILE_REF_COST);
 9173 
 9174   effect(KILL cr);
 9175 
 9176  format %{
 9177     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9178     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9179  %}
 9180 
 9181  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9182             aarch64_enc_cset_eq(res));
 9183 
 9184   ins_pipe(pipe_slow);
 9185 %}
 9186 
 9187 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9188 
 9189   predicate(needs_acquiring_load_exclusive(n));
 9190   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9191   ins_cost(VOLATILE_REF_COST);
 9192 
 9193   effect(KILL cr);
 9194 
 9195  format %{
 9196     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9197     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9198  %}
 9199 
 9200  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9201             aarch64_enc_cset_eq(res));
 9202 
 9203   ins_pipe(pipe_slow);
 9204 %}
 9205 
 9206 
 9207 // ---------------------------------------------------------------------
 9208 
 9209 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9210 
 9211 // Sundry CAS operations.  Note that release is always true,
 9212 // regardless of the memory ordering of the CAS.  This is because we
 9213 // need the volatile case to be sequentially consistent but there is
 9214 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9215 // can't check the type of memory ordering here, so we always emit a
 9216 // STLXR.
 9217 
 9218 // This section is generated from aarch64_ad_cas.m4
 9219 
 9220 
 9221 
 9222 // This pattern is generated automatically from cas.m4.
 9223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9224 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9225 
 9226   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9227   ins_cost(2 * VOLATILE_REF_COST);
 9228   effect(TEMP_DEF res, KILL cr);
 9229   format %{
 9230     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9231   %}
 9232   ins_encode %{
 9233     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9234                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9235                /*weak*/ false, $res$$Register);
 9236     __ sxtbw($res$$Register, $res$$Register);
 9237   %}
 9238   ins_pipe(pipe_slow);
 9239 %}
 9240 
 9241 // This pattern is generated automatically from cas.m4.
 9242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9243 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9244 
 9245   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9246   ins_cost(2 * VOLATILE_REF_COST);
 9247   effect(TEMP_DEF res, KILL cr);
 9248   format %{
 9249     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9250   %}
 9251   ins_encode %{
 9252     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9253                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9254                /*weak*/ false, $res$$Register);
 9255     __ sxthw($res$$Register, $res$$Register);
 9256   %}
 9257   ins_pipe(pipe_slow);
 9258 %}
 9259 
 9260 // This pattern is generated automatically from cas.m4.
 9261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9262 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9263 
 9264   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9265   ins_cost(2 * VOLATILE_REF_COST);
 9266   effect(TEMP_DEF res, KILL cr);
 9267   format %{
 9268     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9269   %}
 9270   ins_encode %{
 9271     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9272                Assembler::word, /*acquire*/ false, /*release*/ true,
 9273                /*weak*/ false, $res$$Register);
 9274   %}
 9275   ins_pipe(pipe_slow);
 9276 %}
 9277 
 9278 // This pattern is generated automatically from cas.m4.
 9279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9280 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9281 
 9282   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9283   ins_cost(2 * VOLATILE_REF_COST);
 9284   effect(TEMP_DEF res, KILL cr);
 9285   format %{
 9286     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9287   %}
 9288   ins_encode %{
 9289     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9290                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9291                /*weak*/ false, $res$$Register);
 9292   %}
 9293   ins_pipe(pipe_slow);
 9294 %}
 9295 
 9296 // This pattern is generated automatically from cas.m4.
 9297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9298 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9299 
 9300   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9301   ins_cost(2 * VOLATILE_REF_COST);
 9302   effect(TEMP_DEF res, KILL cr);
 9303   format %{
 9304     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9305   %}
 9306   ins_encode %{
 9307     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9308                Assembler::word, /*acquire*/ false, /*release*/ true,
 9309                /*weak*/ false, $res$$Register);
 9310   %}
 9311   ins_pipe(pipe_slow);
 9312 %}
 9313 
 9314 // This pattern is generated automatically from cas.m4.
 9315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9316 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9317   predicate(n->as_LoadStore()->barrier_data() == 0);
 9318   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9319   ins_cost(2 * VOLATILE_REF_COST);
 9320   effect(TEMP_DEF res, KILL cr);
 9321   format %{
 9322     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9323   %}
 9324   ins_encode %{
 9325     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9326                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9327                /*weak*/ false, $res$$Register);
 9328   %}
 9329   ins_pipe(pipe_slow);
 9330 %}
 9331 
 9332 // This pattern is generated automatically from cas.m4.
 9333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9334 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9335   predicate(needs_acquiring_load_exclusive(n));
 9336   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9337   ins_cost(VOLATILE_REF_COST);
 9338   effect(TEMP_DEF res, KILL cr);
 9339   format %{
 9340     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9341   %}
 9342   ins_encode %{
 9343     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9344                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9345                /*weak*/ false, $res$$Register);
 9346     __ sxtbw($res$$Register, $res$$Register);
 9347   %}
 9348   ins_pipe(pipe_slow);
 9349 %}
 9350 
 9351 // This pattern is generated automatically from cas.m4.
 9352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9353 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9354   predicate(needs_acquiring_load_exclusive(n));
 9355   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9356   ins_cost(VOLATILE_REF_COST);
 9357   effect(TEMP_DEF res, KILL cr);
 9358   format %{
 9359     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9360   %}
 9361   ins_encode %{
 9362     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9363                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9364                /*weak*/ false, $res$$Register);
 9365     __ sxthw($res$$Register, $res$$Register);
 9366   %}
 9367   ins_pipe(pipe_slow);
 9368 %}
 9369 
 9370 // This pattern is generated automatically from cas.m4.
 9371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9372 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9373   predicate(needs_acquiring_load_exclusive(n));
 9374   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9375   ins_cost(VOLATILE_REF_COST);
 9376   effect(TEMP_DEF res, KILL cr);
 9377   format %{
 9378     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9379   %}
 9380   ins_encode %{
 9381     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9382                Assembler::word, /*acquire*/ true, /*release*/ true,
 9383                /*weak*/ false, $res$$Register);
 9384   %}
 9385   ins_pipe(pipe_slow);
 9386 %}
 9387 
 9388 // This pattern is generated automatically from cas.m4.
 9389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9390 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9391   predicate(needs_acquiring_load_exclusive(n));
 9392   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9393   ins_cost(VOLATILE_REF_COST);
 9394   effect(TEMP_DEF res, KILL cr);
 9395   format %{
 9396     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9397   %}
 9398   ins_encode %{
 9399     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9400                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9401                /*weak*/ false, $res$$Register);
 9402   %}
 9403   ins_pipe(pipe_slow);
 9404 %}
 9405 
 9406 // This pattern is generated automatically from cas.m4.
 9407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9408 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9409   predicate(needs_acquiring_load_exclusive(n));
 9410   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9411   ins_cost(VOLATILE_REF_COST);
 9412   effect(TEMP_DEF res, KILL cr);
 9413   format %{
 9414     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9415   %}
 9416   ins_encode %{
 9417     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9418                Assembler::word, /*acquire*/ true, /*release*/ true,
 9419                /*weak*/ false, $res$$Register);
 9420   %}
 9421   ins_pipe(pipe_slow);
 9422 %}
 9423 
 9424 // This pattern is generated automatically from cas.m4.
 9425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9426 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9427   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9428   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9429   ins_cost(VOLATILE_REF_COST);
 9430   effect(TEMP_DEF res, KILL cr);
 9431   format %{
 9432     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9433   %}
 9434   ins_encode %{
 9435     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9436                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9437                /*weak*/ false, $res$$Register);
 9438   %}
 9439   ins_pipe(pipe_slow);
 9440 %}
 9441 
 9442 // This pattern is generated automatically from cas.m4.
 9443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9444 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9445 
 9446   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9447   ins_cost(2 * VOLATILE_REF_COST);
 9448   effect(KILL cr);
 9449   format %{
 9450     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9451     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9452   %}
 9453   ins_encode %{
 9454     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9455                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9456                /*weak*/ true, noreg);
 9457     __ csetw($res$$Register, Assembler::EQ);
 9458   %}
 9459   ins_pipe(pipe_slow);
 9460 %}
 9461 
 9462 // This pattern is generated automatically from cas.m4.
 9463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9464 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9465 
 9466   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9467   ins_cost(2 * VOLATILE_REF_COST);
 9468   effect(KILL cr);
 9469   format %{
 9470     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9471     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9472   %}
 9473   ins_encode %{
 9474     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9475                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9476                /*weak*/ true, noreg);
 9477     __ csetw($res$$Register, Assembler::EQ);
 9478   %}
 9479   ins_pipe(pipe_slow);
 9480 %}
 9481 
 9482 // This pattern is generated automatically from cas.m4.
 9483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9484 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9485 
 9486   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9487   ins_cost(2 * VOLATILE_REF_COST);
 9488   effect(KILL cr);
 9489   format %{
 9490     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9491     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9492   %}
 9493   ins_encode %{
 9494     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9495                Assembler::word, /*acquire*/ false, /*release*/ true,
 9496                /*weak*/ true, noreg);
 9497     __ csetw($res$$Register, Assembler::EQ);
 9498   %}
 9499   ins_pipe(pipe_slow);
 9500 %}
 9501 
 9502 // This pattern is generated automatically from cas.m4.
 9503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9504 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9505 
 9506   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9507   ins_cost(2 * VOLATILE_REF_COST);
 9508   effect(KILL cr);
 9509   format %{
 9510     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9511     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9512   %}
 9513   ins_encode %{
 9514     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9515                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9516                /*weak*/ true, noreg);
 9517     __ csetw($res$$Register, Assembler::EQ);
 9518   %}
 9519   ins_pipe(pipe_slow);
 9520 %}
 9521 
 9522 // This pattern is generated automatically from cas.m4.
 9523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9524 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9525 
 9526   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9527   ins_cost(2 * VOLATILE_REF_COST);
 9528   effect(KILL cr);
 9529   format %{
 9530     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9531     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9532   %}
 9533   ins_encode %{
 9534     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9535                Assembler::word, /*acquire*/ false, /*release*/ true,
 9536                /*weak*/ true, noreg);
 9537     __ csetw($res$$Register, Assembler::EQ);
 9538   %}
 9539   ins_pipe(pipe_slow);
 9540 %}
 9541 
 9542 // This pattern is generated automatically from cas.m4.
 9543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9544 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9545   predicate(n->as_LoadStore()->barrier_data() == 0);
 9546   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9547   ins_cost(2 * VOLATILE_REF_COST);
 9548   effect(KILL cr);
 9549   format %{
 9550     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9551     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9552   %}
 9553   ins_encode %{
 9554     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9555                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9556                /*weak*/ true, noreg);
 9557     __ csetw($res$$Register, Assembler::EQ);
 9558   %}
 9559   ins_pipe(pipe_slow);
 9560 %}
 9561 
 9562 // This pattern is generated automatically from cas.m4.
 9563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9564 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9565   predicate(needs_acquiring_load_exclusive(n));
 9566   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9567   ins_cost(VOLATILE_REF_COST);
 9568   effect(KILL cr);
 9569   format %{
 9570     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9571     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9572   %}
 9573   ins_encode %{
 9574     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9575                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9576                /*weak*/ true, noreg);
 9577     __ csetw($res$$Register, Assembler::EQ);
 9578   %}
 9579   ins_pipe(pipe_slow);
 9580 %}
 9581 
 9582 // This pattern is generated automatically from cas.m4.
 9583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9584 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9585   predicate(needs_acquiring_load_exclusive(n));
 9586   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9587   ins_cost(VOLATILE_REF_COST);
 9588   effect(KILL cr);
 9589   format %{
 9590     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9591     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9592   %}
 9593   ins_encode %{
 9594     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9595                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9596                /*weak*/ true, noreg);
 9597     __ csetw($res$$Register, Assembler::EQ);
 9598   %}
 9599   ins_pipe(pipe_slow);
 9600 %}
 9601 
 9602 // This pattern is generated automatically from cas.m4.
 9603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9604 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9605   predicate(needs_acquiring_load_exclusive(n));
 9606   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9607   ins_cost(VOLATILE_REF_COST);
 9608   effect(KILL cr);
 9609   format %{
 9610     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9611     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9612   %}
 9613   ins_encode %{
 9614     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9615                Assembler::word, /*acquire*/ true, /*release*/ true,
 9616                /*weak*/ true, noreg);
 9617     __ csetw($res$$Register, Assembler::EQ);
 9618   %}
 9619   ins_pipe(pipe_slow);
 9620 %}
 9621 
 9622 // This pattern is generated automatically from cas.m4.
 9623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9624 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9625   predicate(needs_acquiring_load_exclusive(n));
 9626   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9627   ins_cost(VOLATILE_REF_COST);
 9628   effect(KILL cr);
 9629   format %{
 9630     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9631     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9632   %}
 9633   ins_encode %{
 9634     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9635                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9636                /*weak*/ true, noreg);
 9637     __ csetw($res$$Register, Assembler::EQ);
 9638   %}
 9639   ins_pipe(pipe_slow);
 9640 %}
 9641 
 9642 // This pattern is generated automatically from cas.m4.
 9643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9644 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9645   predicate(needs_acquiring_load_exclusive(n));
 9646   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9647   ins_cost(VOLATILE_REF_COST);
 9648   effect(KILL cr);
 9649   format %{
 9650     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9651     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9652   %}
 9653   ins_encode %{
 9654     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9655                Assembler::word, /*acquire*/ true, /*release*/ true,
 9656                /*weak*/ true, noreg);
 9657     __ csetw($res$$Register, Assembler::EQ);
 9658   %}
 9659   ins_pipe(pipe_slow);
 9660 %}
 9661 
 9662 // This pattern is generated automatically from cas.m4.
 9663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9664 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9665   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9666   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9667   ins_cost(VOLATILE_REF_COST);
 9668   effect(KILL cr);
 9669   format %{
 9670     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9671     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9672   %}
 9673   ins_encode %{
 9674     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9675                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9676                /*weak*/ true, noreg);
 9677     __ csetw($res$$Register, Assembler::EQ);
 9678   %}
 9679   ins_pipe(pipe_slow);
 9680 %}
 9681 
 9682 // END This section of the file is automatically generated. Do not edit --------------
 9683 // ---------------------------------------------------------------------
 9684 
 9685 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9686   match(Set prev (GetAndSetI mem newv));
 9687   ins_cost(2 * VOLATILE_REF_COST);
 9688   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9689   ins_encode %{
 9690     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9691   %}
 9692   ins_pipe(pipe_serial);
 9693 %}
 9694 
 9695 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9696   match(Set prev (GetAndSetL mem newv));
 9697   ins_cost(2 * VOLATILE_REF_COST);
 9698   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9699   ins_encode %{
 9700     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9701   %}
 9702   ins_pipe(pipe_serial);
 9703 %}
 9704 
 9705 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9706   match(Set prev (GetAndSetN mem newv));
 9707   ins_cost(2 * VOLATILE_REF_COST);
 9708   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9709   ins_encode %{
 9710     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9711   %}
 9712   ins_pipe(pipe_serial);
 9713 %}
 9714 
 9715 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9716   predicate(n->as_LoadStore()->barrier_data() == 0);
 9717   match(Set prev (GetAndSetP mem newv));
 9718   ins_cost(2 * VOLATILE_REF_COST);
 9719   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9720   ins_encode %{
 9721     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9722   %}
 9723   ins_pipe(pipe_serial);
 9724 %}
 9725 
 9726 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9727   predicate(needs_acquiring_load_exclusive(n));
 9728   match(Set prev (GetAndSetI mem newv));
 9729   ins_cost(VOLATILE_REF_COST);
 9730   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9731   ins_encode %{
 9732     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9733   %}
 9734   ins_pipe(pipe_serial);
 9735 %}
 9736 
 9737 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9738   predicate(needs_acquiring_load_exclusive(n));
 9739   match(Set prev (GetAndSetL mem newv));
 9740   ins_cost(VOLATILE_REF_COST);
 9741   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9742   ins_encode %{
 9743     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9744   %}
 9745   ins_pipe(pipe_serial);
 9746 %}
 9747 
 9748 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9749   predicate(needs_acquiring_load_exclusive(n));
 9750   match(Set prev (GetAndSetN mem newv));
 9751   ins_cost(VOLATILE_REF_COST);
 9752   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9753   ins_encode %{
 9754     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9755   %}
 9756   ins_pipe(pipe_serial);
 9757 %}
 9758 
 9759 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9760   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9761   match(Set prev (GetAndSetP mem newv));
 9762   ins_cost(VOLATILE_REF_COST);
 9763   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9764   ins_encode %{
 9765     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9766   %}
 9767   ins_pipe(pipe_serial);
 9768 %}
 9769 
 9770 
 9771 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9772   match(Set newval (GetAndAddL mem incr));
 9773   ins_cost(2 * VOLATILE_REF_COST + 1);
 9774   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9775   ins_encode %{
 9776     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9777   %}
 9778   ins_pipe(pipe_serial);
 9779 %}
 9780 
 9781 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9782   predicate(n->as_LoadStore()->result_not_used());
 9783   match(Set dummy (GetAndAddL mem incr));
 9784   ins_cost(2 * VOLATILE_REF_COST);
 9785   format %{ "get_and_addL [$mem], $incr" %}
 9786   ins_encode %{
 9787     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9788   %}
 9789   ins_pipe(pipe_serial);
 9790 %}
 9791 
 9792 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9793   match(Set newval (GetAndAddL mem incr));
 9794   ins_cost(2 * VOLATILE_REF_COST + 1);
 9795   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9796   ins_encode %{
 9797     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9798   %}
 9799   ins_pipe(pipe_serial);
 9800 %}
 9801 
 9802 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9803   predicate(n->as_LoadStore()->result_not_used());
 9804   match(Set dummy (GetAndAddL mem incr));
 9805   ins_cost(2 * VOLATILE_REF_COST);
 9806   format %{ "get_and_addL [$mem], $incr" %}
 9807   ins_encode %{
 9808     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9809   %}
 9810   ins_pipe(pipe_serial);
 9811 %}
 9812 
 9813 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9814   match(Set newval (GetAndAddI mem incr));
 9815   ins_cost(2 * VOLATILE_REF_COST + 1);
 9816   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9817   ins_encode %{
 9818     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9819   %}
 9820   ins_pipe(pipe_serial);
 9821 %}
 9822 
 9823 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9824   predicate(n->as_LoadStore()->result_not_used());
 9825   match(Set dummy (GetAndAddI mem incr));
 9826   ins_cost(2 * VOLATILE_REF_COST);
 9827   format %{ "get_and_addI [$mem], $incr" %}
 9828   ins_encode %{
 9829     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9830   %}
 9831   ins_pipe(pipe_serial);
 9832 %}
 9833 
 9834 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9835   match(Set newval (GetAndAddI mem incr));
 9836   ins_cost(2 * VOLATILE_REF_COST + 1);
 9837   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9838   ins_encode %{
 9839     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9840   %}
 9841   ins_pipe(pipe_serial);
 9842 %}
 9843 
 9844 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9845   predicate(n->as_LoadStore()->result_not_used());
 9846   match(Set dummy (GetAndAddI mem incr));
 9847   ins_cost(2 * VOLATILE_REF_COST);
 9848   format %{ "get_and_addI [$mem], $incr" %}
 9849   ins_encode %{
 9850     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9851   %}
 9852   ins_pipe(pipe_serial);
 9853 %}
 9854 
 9855 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9856   predicate(needs_acquiring_load_exclusive(n));
 9857   match(Set newval (GetAndAddL mem incr));
 9858   ins_cost(VOLATILE_REF_COST + 1);
 9859   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9860   ins_encode %{
 9861     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9862   %}
 9863   ins_pipe(pipe_serial);
 9864 %}
 9865 
 9866 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9867   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9868   match(Set dummy (GetAndAddL mem incr));
 9869   ins_cost(VOLATILE_REF_COST);
 9870   format %{ "get_and_addL_acq [$mem], $incr" %}
 9871   ins_encode %{
 9872     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9873   %}
 9874   ins_pipe(pipe_serial);
 9875 %}
 9876 
 9877 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9878   predicate(needs_acquiring_load_exclusive(n));
 9879   match(Set newval (GetAndAddL mem incr));
 9880   ins_cost(VOLATILE_REF_COST + 1);
 9881   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9882   ins_encode %{
 9883     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9884   %}
 9885   ins_pipe(pipe_serial);
 9886 %}
 9887 
 9888 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9889   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9890   match(Set dummy (GetAndAddL mem incr));
 9891   ins_cost(VOLATILE_REF_COST);
 9892   format %{ "get_and_addL_acq [$mem], $incr" %}
 9893   ins_encode %{
 9894     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9895   %}
 9896   ins_pipe(pipe_serial);
 9897 %}
 9898 
 9899 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9900   predicate(needs_acquiring_load_exclusive(n));
 9901   match(Set newval (GetAndAddI mem incr));
 9902   ins_cost(VOLATILE_REF_COST + 1);
 9903   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9904   ins_encode %{
 9905     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9906   %}
 9907   ins_pipe(pipe_serial);
 9908 %}
 9909 
 9910 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9911   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9912   match(Set dummy (GetAndAddI mem incr));
 9913   ins_cost(VOLATILE_REF_COST);
 9914   format %{ "get_and_addI_acq [$mem], $incr" %}
 9915   ins_encode %{
 9916     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9917   %}
 9918   ins_pipe(pipe_serial);
 9919 %}
 9920 
 9921 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9922   predicate(needs_acquiring_load_exclusive(n));
 9923   match(Set newval (GetAndAddI mem incr));
 9924   ins_cost(VOLATILE_REF_COST + 1);
 9925   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9926   ins_encode %{
 9927     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9928   %}
 9929   ins_pipe(pipe_serial);
 9930 %}
 9931 
 9932 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9933   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9934   match(Set dummy (GetAndAddI mem incr));
 9935   ins_cost(VOLATILE_REF_COST);
 9936   format %{ "get_and_addI_acq [$mem], $incr" %}
 9937   ins_encode %{
 9938     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9939   %}
 9940   ins_pipe(pipe_serial);
 9941 %}
 9942 
 9943 // Manifest a CmpL result in an integer register.
 9944 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9945 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9946 %{
 9947   match(Set dst (CmpL3 src1 src2));
 9948   effect(KILL flags);
 9949 
 9950   ins_cost(INSN_COST * 6);
 9951   format %{
 9952       "cmp $src1, $src2"
 9953       "csetw $dst, ne"
 9954       "cnegw $dst, lt"
 9955   %}
 9956   // format %{ "CmpL3 $dst, $src1, $src2" %}
 9957   ins_encode %{
 9958     __ cmp($src1$$Register, $src2$$Register);
 9959     __ csetw($dst$$Register, Assembler::NE);
 9960     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9961   %}
 9962 
 9963   ins_pipe(pipe_class_default);
 9964 %}
 9965 
 9966 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9967 %{
 9968   match(Set dst (CmpL3 src1 src2));
 9969   effect(KILL flags);
 9970 
 9971   ins_cost(INSN_COST * 6);
 9972   format %{
 9973       "cmp $src1, $src2"
 9974       "csetw $dst, ne"
 9975       "cnegw $dst, lt"
 9976   %}
 9977   ins_encode %{
 9978     int32_t con = (int32_t)$src2$$constant;
 9979      if (con < 0) {
 9980       __ adds(zr, $src1$$Register, -con);
 9981     } else {
 9982       __ subs(zr, $src1$$Register, con);
 9983     }
 9984     __ csetw($dst$$Register, Assembler::NE);
 9985     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9986   %}
 9987 
 9988   ins_pipe(pipe_class_default);
 9989 %}
 9990 
 9991 // ============================================================================
 9992 // Conditional Move Instructions
 9993 
 9994 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9995 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9996 // define an op class which merged both inputs and use it to type the
 9997 // argument to a single rule. unfortunatelyt his fails because the
 9998 // opclass does not live up to the COND_INTER interface of its
 9999 // component operands. When the generic code tries to negate the
10000 // operand it ends up running the generci Machoper::negate method
10001 // which throws a ShouldNotHappen. So, we have to provide two flavours
10002 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10003 
10004 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10005   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10006 
10007   ins_cost(INSN_COST * 2);
10008   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10009 
10010   ins_encode %{
10011     __ cselw(as_Register($dst$$reg),
10012              as_Register($src2$$reg),
10013              as_Register($src1$$reg),
10014              (Assembler::Condition)$cmp$$cmpcode);
10015   %}
10016 
10017   ins_pipe(icond_reg_reg);
10018 %}
10019 
10020 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10021   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10022 
10023   ins_cost(INSN_COST * 2);
10024   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10025 
10026   ins_encode %{
10027     __ cselw(as_Register($dst$$reg),
10028              as_Register($src2$$reg),
10029              as_Register($src1$$reg),
10030              (Assembler::Condition)$cmp$$cmpcode);
10031   %}
10032 
10033   ins_pipe(icond_reg_reg);
10034 %}
10035 
10036 // special cases where one arg is zero
10037 
10038 // n.b. this is selected in preference to the rule above because it
10039 // avoids loading constant 0 into a source register
10040 
10041 // TODO
10042 // we ought only to be able to cull one of these variants as the ideal
10043 // transforms ought always to order the zero consistently (to left/right?)
10044 
10045 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10046   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10047 
10048   ins_cost(INSN_COST * 2);
10049   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10050 
10051   ins_encode %{
10052     __ cselw(as_Register($dst$$reg),
10053              as_Register($src$$reg),
10054              zr,
10055              (Assembler::Condition)$cmp$$cmpcode);
10056   %}
10057 
10058   ins_pipe(icond_reg);
10059 %}
10060 
10061 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10062   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10063 
10064   ins_cost(INSN_COST * 2);
10065   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10066 
10067   ins_encode %{
10068     __ cselw(as_Register($dst$$reg),
10069              as_Register($src$$reg),
10070              zr,
10071              (Assembler::Condition)$cmp$$cmpcode);
10072   %}
10073 
10074   ins_pipe(icond_reg);
10075 %}
10076 
10077 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10078   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10079 
10080   ins_cost(INSN_COST * 2);
10081   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10082 
10083   ins_encode %{
10084     __ cselw(as_Register($dst$$reg),
10085              zr,
10086              as_Register($src$$reg),
10087              (Assembler::Condition)$cmp$$cmpcode);
10088   %}
10089 
10090   ins_pipe(icond_reg);
10091 %}
10092 
10093 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10094   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10095 
10096   ins_cost(INSN_COST * 2);
10097   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10098 
10099   ins_encode %{
10100     __ cselw(as_Register($dst$$reg),
10101              zr,
10102              as_Register($src$$reg),
10103              (Assembler::Condition)$cmp$$cmpcode);
10104   %}
10105 
10106   ins_pipe(icond_reg);
10107 %}
10108 
10109 // special case for creating a boolean 0 or 1
10110 
10111 // n.b. this is selected in preference to the rule above because it
10112 // avoids loading constants 0 and 1 into a source register
10113 
10114 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10115   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10116 
10117   ins_cost(INSN_COST * 2);
10118   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10119 
10120   ins_encode %{
10121     // equivalently
10122     // cset(as_Register($dst$$reg),
10123     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10124     __ csincw(as_Register($dst$$reg),
10125              zr,
10126              zr,
10127              (Assembler::Condition)$cmp$$cmpcode);
10128   %}
10129 
10130   ins_pipe(icond_none);
10131 %}
10132 
10133 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10134   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10135 
10136   ins_cost(INSN_COST * 2);
10137   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10138 
10139   ins_encode %{
10140     // equivalently
10141     // cset(as_Register($dst$$reg),
10142     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10143     __ csincw(as_Register($dst$$reg),
10144              zr,
10145              zr,
10146              (Assembler::Condition)$cmp$$cmpcode);
10147   %}
10148 
10149   ins_pipe(icond_none);
10150 %}
10151 
10152 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10153   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10154 
10155   ins_cost(INSN_COST * 2);
10156   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10157 
10158   ins_encode %{
10159     __ csel(as_Register($dst$$reg),
10160             as_Register($src2$$reg),
10161             as_Register($src1$$reg),
10162             (Assembler::Condition)$cmp$$cmpcode);
10163   %}
10164 
10165   ins_pipe(icond_reg_reg);
10166 %}
10167 
10168 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10169   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10170 
10171   ins_cost(INSN_COST * 2);
10172   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10173 
10174   ins_encode %{
10175     __ csel(as_Register($dst$$reg),
10176             as_Register($src2$$reg),
10177             as_Register($src1$$reg),
10178             (Assembler::Condition)$cmp$$cmpcode);
10179   %}
10180 
10181   ins_pipe(icond_reg_reg);
10182 %}
10183 
10184 // special cases where one arg is zero
10185 
10186 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10187   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10188 
10189   ins_cost(INSN_COST * 2);
10190   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10191 
10192   ins_encode %{
10193     __ csel(as_Register($dst$$reg),
10194             zr,
10195             as_Register($src$$reg),
10196             (Assembler::Condition)$cmp$$cmpcode);
10197   %}
10198 
10199   ins_pipe(icond_reg);
10200 %}
10201 
10202 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10203   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10204 
10205   ins_cost(INSN_COST * 2);
10206   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10207 
10208   ins_encode %{
10209     __ csel(as_Register($dst$$reg),
10210             zr,
10211             as_Register($src$$reg),
10212             (Assembler::Condition)$cmp$$cmpcode);
10213   %}
10214 
10215   ins_pipe(icond_reg);
10216 %}
10217 
10218 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10219   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10220 
10221   ins_cost(INSN_COST * 2);
10222   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10223 
10224   ins_encode %{
10225     __ csel(as_Register($dst$$reg),
10226             as_Register($src$$reg),
10227             zr,
10228             (Assembler::Condition)$cmp$$cmpcode);
10229   %}
10230 
10231   ins_pipe(icond_reg);
10232 %}
10233 
10234 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10235   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10236 
10237   ins_cost(INSN_COST * 2);
10238   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10239 
10240   ins_encode %{
10241     __ csel(as_Register($dst$$reg),
10242             as_Register($src$$reg),
10243             zr,
10244             (Assembler::Condition)$cmp$$cmpcode);
10245   %}
10246 
10247   ins_pipe(icond_reg);
10248 %}
10249 
10250 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10251   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10252 
10253   ins_cost(INSN_COST * 2);
10254   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10255 
10256   ins_encode %{
10257     __ csel(as_Register($dst$$reg),
10258             as_Register($src2$$reg),
10259             as_Register($src1$$reg),
10260             (Assembler::Condition)$cmp$$cmpcode);
10261   %}
10262 
10263   ins_pipe(icond_reg_reg);
10264 %}
10265 
10266 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10267   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10268 
10269   ins_cost(INSN_COST * 2);
10270   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10271 
10272   ins_encode %{
10273     __ csel(as_Register($dst$$reg),
10274             as_Register($src2$$reg),
10275             as_Register($src1$$reg),
10276             (Assembler::Condition)$cmp$$cmpcode);
10277   %}
10278 
10279   ins_pipe(icond_reg_reg);
10280 %}
10281 
10282 // special cases where one arg is zero
10283 
10284 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10285   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10286 
10287   ins_cost(INSN_COST * 2);
10288   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10289 
10290   ins_encode %{
10291     __ csel(as_Register($dst$$reg),
10292             zr,
10293             as_Register($src$$reg),
10294             (Assembler::Condition)$cmp$$cmpcode);
10295   %}
10296 
10297   ins_pipe(icond_reg);
10298 %}
10299 
10300 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10301   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10302 
10303   ins_cost(INSN_COST * 2);
10304   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10305 
10306   ins_encode %{
10307     __ csel(as_Register($dst$$reg),
10308             zr,
10309             as_Register($src$$reg),
10310             (Assembler::Condition)$cmp$$cmpcode);
10311   %}
10312 
10313   ins_pipe(icond_reg);
10314 %}
10315 
10316 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10317   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10318 
10319   ins_cost(INSN_COST * 2);
10320   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10321 
10322   ins_encode %{
10323     __ csel(as_Register($dst$$reg),
10324             as_Register($src$$reg),
10325             zr,
10326             (Assembler::Condition)$cmp$$cmpcode);
10327   %}
10328 
10329   ins_pipe(icond_reg);
10330 %}
10331 
10332 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10333   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10334 
10335   ins_cost(INSN_COST * 2);
10336   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10337 
10338   ins_encode %{
10339     __ csel(as_Register($dst$$reg),
10340             as_Register($src$$reg),
10341             zr,
10342             (Assembler::Condition)$cmp$$cmpcode);
10343   %}
10344 
10345   ins_pipe(icond_reg);
10346 %}
10347 
10348 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10349   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10350 
10351   ins_cost(INSN_COST * 2);
10352   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10353 
10354   ins_encode %{
10355     __ cselw(as_Register($dst$$reg),
10356              as_Register($src2$$reg),
10357              as_Register($src1$$reg),
10358              (Assembler::Condition)$cmp$$cmpcode);
10359   %}
10360 
10361   ins_pipe(icond_reg_reg);
10362 %}
10363 
10364 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10365   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10366 
10367   ins_cost(INSN_COST * 2);
10368   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10369 
10370   ins_encode %{
10371     __ cselw(as_Register($dst$$reg),
10372              as_Register($src2$$reg),
10373              as_Register($src1$$reg),
10374              (Assembler::Condition)$cmp$$cmpcode);
10375   %}
10376 
10377   ins_pipe(icond_reg_reg);
10378 %}
10379 
10380 // special cases where one arg is zero
10381 
10382 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10383   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10384 
10385   ins_cost(INSN_COST * 2);
10386   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10387 
10388   ins_encode %{
10389     __ cselw(as_Register($dst$$reg),
10390              zr,
10391              as_Register($src$$reg),
10392              (Assembler::Condition)$cmp$$cmpcode);
10393   %}
10394 
10395   ins_pipe(icond_reg);
10396 %}
10397 
10398 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10399   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10400 
10401   ins_cost(INSN_COST * 2);
10402   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10403 
10404   ins_encode %{
10405     __ cselw(as_Register($dst$$reg),
10406              zr,
10407              as_Register($src$$reg),
10408              (Assembler::Condition)$cmp$$cmpcode);
10409   %}
10410 
10411   ins_pipe(icond_reg);
10412 %}
10413 
10414 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10415   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10416 
10417   ins_cost(INSN_COST * 2);
10418   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10419 
10420   ins_encode %{
10421     __ cselw(as_Register($dst$$reg),
10422              as_Register($src$$reg),
10423              zr,
10424              (Assembler::Condition)$cmp$$cmpcode);
10425   %}
10426 
10427   ins_pipe(icond_reg);
10428 %}
10429 
10430 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10431   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10432 
10433   ins_cost(INSN_COST * 2);
10434   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10435 
10436   ins_encode %{
10437     __ cselw(as_Register($dst$$reg),
10438              as_Register($src$$reg),
10439              zr,
10440              (Assembler::Condition)$cmp$$cmpcode);
10441   %}
10442 
10443   ins_pipe(icond_reg);
10444 %}
10445 
10446 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10447 %{
10448   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10449 
10450   ins_cost(INSN_COST * 3);
10451 
10452   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10453   ins_encode %{
10454     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10455     __ fcsels(as_FloatRegister($dst$$reg),
10456               as_FloatRegister($src2$$reg),
10457               as_FloatRegister($src1$$reg),
10458               cond);
10459   %}
10460 
10461   ins_pipe(fp_cond_reg_reg_s);
10462 %}
10463 
10464 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10465 %{
10466   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10467 
10468   ins_cost(INSN_COST * 3);
10469 
10470   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10471   ins_encode %{
10472     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10473     __ fcsels(as_FloatRegister($dst$$reg),
10474               as_FloatRegister($src2$$reg),
10475               as_FloatRegister($src1$$reg),
10476               cond);
10477   %}
10478 
10479   ins_pipe(fp_cond_reg_reg_s);
10480 %}
10481 
10482 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10483 %{
10484   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10485 
10486   ins_cost(INSN_COST * 3);
10487 
10488   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10489   ins_encode %{
10490     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10491     __ fcseld(as_FloatRegister($dst$$reg),
10492               as_FloatRegister($src2$$reg),
10493               as_FloatRegister($src1$$reg),
10494               cond);
10495   %}
10496 
10497   ins_pipe(fp_cond_reg_reg_d);
10498 %}
10499 
10500 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10501 %{
10502   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10503 
10504   ins_cost(INSN_COST * 3);
10505 
10506   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10507   ins_encode %{
10508     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10509     __ fcseld(as_FloatRegister($dst$$reg),
10510               as_FloatRegister($src2$$reg),
10511               as_FloatRegister($src1$$reg),
10512               cond);
10513   %}
10514 
10515   ins_pipe(fp_cond_reg_reg_d);
10516 %}
10517 
10518 // ============================================================================
10519 // Arithmetic Instructions
10520 //
10521 
10522 // Integer Addition
10523 
10524 // TODO
10525 // these currently employ operations which do not set CR and hence are
10526 // not flagged as killing CR but we would like to isolate the cases
10527 // where we want to set flags from those where we don't. need to work
10528 // out how to do that.
10529 
10530 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10531   match(Set dst (AddI src1 src2));
10532 
10533   ins_cost(INSN_COST);
10534   format %{ "addw  $dst, $src1, $src2" %}
10535 
10536   ins_encode %{
10537     __ addw(as_Register($dst$$reg),
10538             as_Register($src1$$reg),
10539             as_Register($src2$$reg));
10540   %}
10541 
10542   ins_pipe(ialu_reg_reg);
10543 %}
10544 
10545 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10546   match(Set dst (AddI src1 src2));
10547 
10548   ins_cost(INSN_COST);
10549   format %{ "addw $dst, $src1, $src2" %}
10550 
10551   // use opcode to indicate that this is an add not a sub
10552   opcode(0x0);
10553 
10554   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10555 
10556   ins_pipe(ialu_reg_imm);
10557 %}
10558 
10559 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10560   match(Set dst (AddI (ConvL2I src1) src2));
10561 
10562   ins_cost(INSN_COST);
10563   format %{ "addw $dst, $src1, $src2" %}
10564 
10565   // use opcode to indicate that this is an add not a sub
10566   opcode(0x0);
10567 
10568   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10569 
10570   ins_pipe(ialu_reg_imm);
10571 %}
10572 
10573 // Pointer Addition
10574 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10575   match(Set dst (AddP src1 src2));
10576 
10577   ins_cost(INSN_COST);
10578   format %{ "add $dst, $src1, $src2\t# ptr" %}
10579 
10580   ins_encode %{
10581     __ add(as_Register($dst$$reg),
10582            as_Register($src1$$reg),
10583            as_Register($src2$$reg));
10584   %}
10585 
10586   ins_pipe(ialu_reg_reg);
10587 %}
10588 
10589 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10590   match(Set dst (AddP src1 (ConvI2L src2)));
10591 
10592   ins_cost(1.9 * INSN_COST);
10593   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10594 
10595   ins_encode %{
10596     __ add(as_Register($dst$$reg),
10597            as_Register($src1$$reg),
10598            as_Register($src2$$reg), ext::sxtw);
10599   %}
10600 
10601   ins_pipe(ialu_reg_reg);
10602 %}
10603 
10604 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10605   match(Set dst (AddP src1 (LShiftL src2 scale)));
10606 
10607   ins_cost(1.9 * INSN_COST);
10608   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10609 
10610   ins_encode %{
10611     __ lea(as_Register($dst$$reg),
10612            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10613                    Address::lsl($scale$$constant)));
10614   %}
10615 
10616   ins_pipe(ialu_reg_reg_shift);
10617 %}
10618 
10619 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10620   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10621 
10622   ins_cost(1.9 * INSN_COST);
10623   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10624 
10625   ins_encode %{
10626     __ lea(as_Register($dst$$reg),
10627            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10628                    Address::sxtw($scale$$constant)));
10629   %}
10630 
10631   ins_pipe(ialu_reg_reg_shift);
10632 %}
10633 
10634 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10635   match(Set dst (LShiftL (ConvI2L src) scale));
10636 
10637   ins_cost(INSN_COST);
10638   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10639 
10640   ins_encode %{
10641     __ sbfiz(as_Register($dst$$reg),
10642           as_Register($src$$reg),
10643           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10644   %}
10645 
10646   ins_pipe(ialu_reg_shift);
10647 %}
10648 
10649 // Pointer Immediate Addition
10650 // n.b. this needs to be more expensive than using an indirect memory
10651 // operand
10652 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10653   match(Set dst (AddP src1 src2));
10654 
10655   ins_cost(INSN_COST);
10656   format %{ "add $dst, $src1, $src2\t# ptr" %}
10657 
10658   // use opcode to indicate that this is an add not a sub
10659   opcode(0x0);
10660 
10661   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10662 
10663   ins_pipe(ialu_reg_imm);
10664 %}
10665 
10666 // Long Addition
10667 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10668 
10669   match(Set dst (AddL src1 src2));
10670 
10671   ins_cost(INSN_COST);
10672   format %{ "add  $dst, $src1, $src2" %}
10673 
10674   ins_encode %{
10675     __ add(as_Register($dst$$reg),
10676            as_Register($src1$$reg),
10677            as_Register($src2$$reg));
10678   %}
10679 
10680   ins_pipe(ialu_reg_reg);
10681 %}
10682 
10683 // No constant pool entries requiredLong Immediate Addition.
10684 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10685   match(Set dst (AddL src1 src2));
10686 
10687   ins_cost(INSN_COST);
10688   format %{ "add $dst, $src1, $src2" %}
10689 
10690   // use opcode to indicate that this is an add not a sub
10691   opcode(0x0);
10692 
10693   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10694 
10695   ins_pipe(ialu_reg_imm);
10696 %}
10697 
10698 // Integer Subtraction
10699 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10700   match(Set dst (SubI src1 src2));
10701 
10702   ins_cost(INSN_COST);
10703   format %{ "subw  $dst, $src1, $src2" %}
10704 
10705   ins_encode %{
10706     __ subw(as_Register($dst$$reg),
10707             as_Register($src1$$reg),
10708             as_Register($src2$$reg));
10709   %}
10710 
10711   ins_pipe(ialu_reg_reg);
10712 %}
10713 
10714 // Immediate Subtraction
10715 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10716   match(Set dst (SubI src1 src2));
10717 
10718   ins_cost(INSN_COST);
10719   format %{ "subw $dst, $src1, $src2" %}
10720 
10721   // use opcode to indicate that this is a sub not an add
10722   opcode(0x1);
10723 
10724   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10725 
10726   ins_pipe(ialu_reg_imm);
10727 %}
10728 
10729 // Long Subtraction
10730 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10731 
10732   match(Set dst (SubL src1 src2));
10733 
10734   ins_cost(INSN_COST);
10735   format %{ "sub  $dst, $src1, $src2" %}
10736 
10737   ins_encode %{
10738     __ sub(as_Register($dst$$reg),
10739            as_Register($src1$$reg),
10740            as_Register($src2$$reg));
10741   %}
10742 
10743   ins_pipe(ialu_reg_reg);
10744 %}
10745 
10746 // No constant pool entries requiredLong Immediate Subtraction.
10747 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10748   match(Set dst (SubL src1 src2));
10749 
10750   ins_cost(INSN_COST);
10751   format %{ "sub$dst, $src1, $src2" %}
10752 
10753   // use opcode to indicate that this is a sub not an add
10754   opcode(0x1);
10755 
10756   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10757 
10758   ins_pipe(ialu_reg_imm);
10759 %}
10760 
10761 // Integer Negation (special case for sub)
10762 
10763 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10764   match(Set dst (SubI zero src));
10765 
10766   ins_cost(INSN_COST);
10767   format %{ "negw $dst, $src\t# int" %}
10768 
10769   ins_encode %{
10770     __ negw(as_Register($dst$$reg),
10771             as_Register($src$$reg));
10772   %}
10773 
10774   ins_pipe(ialu_reg);
10775 %}
10776 
10777 // Long Negation
10778 
10779 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10780   match(Set dst (SubL zero src));
10781 
10782   ins_cost(INSN_COST);
10783   format %{ "neg $dst, $src\t# long" %}
10784 
10785   ins_encode %{
10786     __ neg(as_Register($dst$$reg),
10787            as_Register($src$$reg));
10788   %}
10789 
10790   ins_pipe(ialu_reg);
10791 %}
10792 
10793 // Integer Multiply
10794 
10795 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10796   match(Set dst (MulI src1 src2));
10797 
10798   ins_cost(INSN_COST * 3);
10799   format %{ "mulw  $dst, $src1, $src2" %}
10800 
10801   ins_encode %{
10802     __ mulw(as_Register($dst$$reg),
10803             as_Register($src1$$reg),
10804             as_Register($src2$$reg));
10805   %}
10806 
10807   ins_pipe(imul_reg_reg);
10808 %}
10809 
10810 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10811   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10812 
10813   ins_cost(INSN_COST * 3);
10814   format %{ "smull  $dst, $src1, $src2" %}
10815 
10816   ins_encode %{
10817     __ smull(as_Register($dst$$reg),
10818              as_Register($src1$$reg),
10819              as_Register($src2$$reg));
10820   %}
10821 
10822   ins_pipe(imul_reg_reg);
10823 %}
10824 
10825 // Long Multiply
10826 
10827 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10828   match(Set dst (MulL src1 src2));
10829 
10830   ins_cost(INSN_COST * 5);
10831   format %{ "mul  $dst, $src1, $src2" %}
10832 
10833   ins_encode %{
10834     __ mul(as_Register($dst$$reg),
10835            as_Register($src1$$reg),
10836            as_Register($src2$$reg));
10837   %}
10838 
10839   ins_pipe(lmul_reg_reg);
10840 %}
10841 
10842 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10843 %{
10844   match(Set dst (MulHiL src1 src2));
10845 
10846   ins_cost(INSN_COST * 7);
10847   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10848 
10849   ins_encode %{
10850     __ smulh(as_Register($dst$$reg),
10851              as_Register($src1$$reg),
10852              as_Register($src2$$reg));
10853   %}
10854 
10855   ins_pipe(lmul_reg_reg);
10856 %}
10857 
10858 // Combined Integer Multiply & Add/Sub
10859 
10860 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10861   match(Set dst (AddI src3 (MulI src1 src2)));
10862 
10863   ins_cost(INSN_COST * 3);
10864   format %{ "madd  $dst, $src1, $src2, $src3" %}
10865 
10866   ins_encode %{
10867     __ maddw(as_Register($dst$$reg),
10868              as_Register($src1$$reg),
10869              as_Register($src2$$reg),
10870              as_Register($src3$$reg));
10871   %}
10872 
10873   ins_pipe(imac_reg_reg);
10874 %}
10875 
10876 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10877   match(Set dst (SubI src3 (MulI src1 src2)));
10878 
10879   ins_cost(INSN_COST * 3);
10880   format %{ "msub  $dst, $src1, $src2, $src3" %}
10881 
10882   ins_encode %{
10883     __ msubw(as_Register($dst$$reg),
10884              as_Register($src1$$reg),
10885              as_Register($src2$$reg),
10886              as_Register($src3$$reg));
10887   %}
10888 
10889   ins_pipe(imac_reg_reg);
10890 %}
10891 
10892 // Combined Integer Multiply & Neg
10893 
10894 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10895   match(Set dst (MulI (SubI zero src1) src2));
10896   match(Set dst (MulI src1 (SubI zero src2)));
10897 
10898   ins_cost(INSN_COST * 3);
10899   format %{ "mneg  $dst, $src1, $src2" %}
10900 
10901   ins_encode %{
10902     __ mnegw(as_Register($dst$$reg),
10903              as_Register($src1$$reg),
10904              as_Register($src2$$reg));
10905   %}
10906 
10907   ins_pipe(imac_reg_reg);
10908 %}
10909 
10910 // Combined Long Multiply & Add/Sub
10911 
10912 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10913   match(Set dst (AddL src3 (MulL src1 src2)));
10914 
10915   ins_cost(INSN_COST * 5);
10916   format %{ "madd  $dst, $src1, $src2, $src3" %}
10917 
10918   ins_encode %{
10919     __ madd(as_Register($dst$$reg),
10920             as_Register($src1$$reg),
10921             as_Register($src2$$reg),
10922             as_Register($src3$$reg));
10923   %}
10924 
10925   ins_pipe(lmac_reg_reg);
10926 %}
10927 
10928 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10929   match(Set dst (SubL src3 (MulL src1 src2)));
10930 
10931   ins_cost(INSN_COST * 5);
10932   format %{ "msub  $dst, $src1, $src2, $src3" %}
10933 
10934   ins_encode %{
10935     __ msub(as_Register($dst$$reg),
10936             as_Register($src1$$reg),
10937             as_Register($src2$$reg),
10938             as_Register($src3$$reg));
10939   %}
10940 
10941   ins_pipe(lmac_reg_reg);
10942 %}
10943 
10944 // Combined Long Multiply & Neg
10945 
10946 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10947   match(Set dst (MulL (SubL zero src1) src2));
10948   match(Set dst (MulL src1 (SubL zero src2)));
10949 
10950   ins_cost(INSN_COST * 5);
10951   format %{ "mneg  $dst, $src1, $src2" %}
10952 
10953   ins_encode %{
10954     __ mneg(as_Register($dst$$reg),
10955             as_Register($src1$$reg),
10956             as_Register($src2$$reg));
10957   %}
10958 
10959   ins_pipe(lmac_reg_reg);
10960 %}
10961 
10962 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10963 
10964 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10965   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10966 
10967   ins_cost(INSN_COST * 3);
10968   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10969 
10970   ins_encode %{
10971     __ smaddl(as_Register($dst$$reg),
10972               as_Register($src1$$reg),
10973               as_Register($src2$$reg),
10974               as_Register($src3$$reg));
10975   %}
10976 
10977   ins_pipe(imac_reg_reg);
10978 %}
10979 
10980 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10981   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10982 
10983   ins_cost(INSN_COST * 3);
10984   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10985 
10986   ins_encode %{
10987     __ smsubl(as_Register($dst$$reg),
10988               as_Register($src1$$reg),
10989               as_Register($src2$$reg),
10990               as_Register($src3$$reg));
10991   %}
10992 
10993   ins_pipe(imac_reg_reg);
10994 %}
10995 
10996 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10997   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10998   match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
10999 
11000   ins_cost(INSN_COST * 3);
11001   format %{ "smnegl  $dst, $src1, $src2" %}
11002 
11003   ins_encode %{
11004     __ smnegl(as_Register($dst$$reg),
11005               as_Register($src1$$reg),
11006               as_Register($src2$$reg));
11007   %}
11008 
11009   ins_pipe(imac_reg_reg);
11010 %}
11011 
11012 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11013 
11014 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11015   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11016 
11017   ins_cost(INSN_COST * 5);
11018   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11019             "maddw $dst, $src3, $src4, rscratch1" %}
11020 
11021   ins_encode %{
11022     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11023     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11024 
11025   ins_pipe(imac_reg_reg);
11026 %}
11027 
11028 // Integer Divide
11029 
11030 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11031   match(Set dst (DivI src1 src2));
11032 
11033   ins_cost(INSN_COST * 19);
11034   format %{ "sdivw  $dst, $src1, $src2" %}
11035 
11036   ins_encode(aarch64_enc_divw(dst, src1, src2));
11037   ins_pipe(idiv_reg_reg);
11038 %}
11039 
11040 // Long Divide
11041 
11042 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11043   match(Set dst (DivL src1 src2));
11044 
11045   ins_cost(INSN_COST * 35);
11046   format %{ "sdiv   $dst, $src1, $src2" %}
11047 
11048   ins_encode(aarch64_enc_div(dst, src1, src2));
11049   ins_pipe(ldiv_reg_reg);
11050 %}
11051 
11052 // Integer Remainder
11053 
11054 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11055   match(Set dst (ModI src1 src2));
11056 
11057   ins_cost(INSN_COST * 22);
11058   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11059             "msubw($dst, rscratch1, $src2, $src1" %}
11060 
11061   ins_encode(aarch64_enc_modw(dst, src1, src2));
11062   ins_pipe(idiv_reg_reg);
11063 %}
11064 
11065 // Long Remainder
11066 
11067 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11068   match(Set dst (ModL src1 src2));
11069 
11070   ins_cost(INSN_COST * 38);
11071   format %{ "sdiv   rscratch1, $src1, $src2\n"
11072             "msub($dst, rscratch1, $src2, $src1" %}
11073 
11074   ins_encode(aarch64_enc_mod(dst, src1, src2));
11075   ins_pipe(ldiv_reg_reg);
11076 %}
11077 
11078 // Integer Shifts
11079 
11080 // Shift Left Register
11081 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11082   match(Set dst (LShiftI src1 src2));
11083 
11084   ins_cost(INSN_COST * 2);
11085   format %{ "lslvw  $dst, $src1, $src2" %}
11086 
11087   ins_encode %{
11088     __ lslvw(as_Register($dst$$reg),
11089              as_Register($src1$$reg),
11090              as_Register($src2$$reg));
11091   %}
11092 
11093   ins_pipe(ialu_reg_reg_vshift);
11094 %}
11095 
11096 // Shift Left Immediate
11097 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11098   match(Set dst (LShiftI src1 src2));
11099 
11100   ins_cost(INSN_COST);
11101   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11102 
11103   ins_encode %{
11104     __ lslw(as_Register($dst$$reg),
11105             as_Register($src1$$reg),
11106             $src2$$constant & 0x1f);
11107   %}
11108 
11109   ins_pipe(ialu_reg_shift);
11110 %}
11111 
11112 // Shift Right Logical Register
11113 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11114   match(Set dst (URShiftI src1 src2));
11115 
11116   ins_cost(INSN_COST * 2);
11117   format %{ "lsrvw  $dst, $src1, $src2" %}
11118 
11119   ins_encode %{
11120     __ lsrvw(as_Register($dst$$reg),
11121              as_Register($src1$$reg),
11122              as_Register($src2$$reg));
11123   %}
11124 
11125   ins_pipe(ialu_reg_reg_vshift);
11126 %}
11127 
11128 // Shift Right Logical Immediate
11129 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11130   match(Set dst (URShiftI src1 src2));
11131 
11132   ins_cost(INSN_COST);
11133   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11134 
11135   ins_encode %{
11136     __ lsrw(as_Register($dst$$reg),
11137             as_Register($src1$$reg),
11138             $src2$$constant & 0x1f);
11139   %}
11140 
11141   ins_pipe(ialu_reg_shift);
11142 %}
11143 
11144 // Shift Right Arithmetic Register
11145 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11146   match(Set dst (RShiftI src1 src2));
11147 
11148   ins_cost(INSN_COST * 2);
11149   format %{ "asrvw  $dst, $src1, $src2" %}
11150 
11151   ins_encode %{
11152     __ asrvw(as_Register($dst$$reg),
11153              as_Register($src1$$reg),
11154              as_Register($src2$$reg));
11155   %}
11156 
11157   ins_pipe(ialu_reg_reg_vshift);
11158 %}
11159 
11160 // Shift Right Arithmetic Immediate
11161 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11162   match(Set dst (RShiftI src1 src2));
11163 
11164   ins_cost(INSN_COST);
11165   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11166 
11167   ins_encode %{
11168     __ asrw(as_Register($dst$$reg),
11169             as_Register($src1$$reg),
11170             $src2$$constant & 0x1f);
11171   %}
11172 
11173   ins_pipe(ialu_reg_shift);
11174 %}
11175 
11176 // Combined Int Mask and Right Shift (using UBFM)
11177 // TODO
11178 
11179 // Long Shifts
11180 
11181 // Shift Left Register
11182 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11183   match(Set dst (LShiftL src1 src2));
11184 
11185   ins_cost(INSN_COST * 2);
11186   format %{ "lslv  $dst, $src1, $src2" %}
11187 
11188   ins_encode %{
11189     __ lslv(as_Register($dst$$reg),
11190             as_Register($src1$$reg),
11191             as_Register($src2$$reg));
11192   %}
11193 
11194   ins_pipe(ialu_reg_reg_vshift);
11195 %}
11196 
11197 // Shift Left Immediate
11198 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11199   match(Set dst (LShiftL src1 src2));
11200 
11201   ins_cost(INSN_COST);
11202   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11203 
11204   ins_encode %{
11205     __ lsl(as_Register($dst$$reg),
11206             as_Register($src1$$reg),
11207             $src2$$constant & 0x3f);
11208   %}
11209 
11210   ins_pipe(ialu_reg_shift);
11211 %}
11212 
11213 // Shift Right Logical Register
11214 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11215   match(Set dst (URShiftL src1 src2));
11216 
11217   ins_cost(INSN_COST * 2);
11218   format %{ "lsrv  $dst, $src1, $src2" %}
11219 
11220   ins_encode %{
11221     __ lsrv(as_Register($dst$$reg),
11222             as_Register($src1$$reg),
11223             as_Register($src2$$reg));
11224   %}
11225 
11226   ins_pipe(ialu_reg_reg_vshift);
11227 %}
11228 
11229 // Shift Right Logical Immediate
11230 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11231   match(Set dst (URShiftL src1 src2));
11232 
11233   ins_cost(INSN_COST);
11234   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11235 
11236   ins_encode %{
11237     __ lsr(as_Register($dst$$reg),
11238            as_Register($src1$$reg),
11239            $src2$$constant & 0x3f);
11240   %}
11241 
11242   ins_pipe(ialu_reg_shift);
11243 %}
11244 
11245 // A special-case pattern for card table stores.
11246 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11247   match(Set dst (URShiftL (CastP2X src1) src2));
11248 
11249   ins_cost(INSN_COST);
11250   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11251 
11252   ins_encode %{
11253     __ lsr(as_Register($dst$$reg),
11254            as_Register($src1$$reg),
11255            $src2$$constant & 0x3f);
11256   %}
11257 
11258   ins_pipe(ialu_reg_shift);
11259 %}
11260 
11261 // Shift Right Arithmetic Register
11262 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11263   match(Set dst (RShiftL src1 src2));
11264 
11265   ins_cost(INSN_COST * 2);
11266   format %{ "asrv  $dst, $src1, $src2" %}
11267 
11268   ins_encode %{
11269     __ asrv(as_Register($dst$$reg),
11270             as_Register($src1$$reg),
11271             as_Register($src2$$reg));
11272   %}
11273 
11274   ins_pipe(ialu_reg_reg_vshift);
11275 %}
11276 
11277 // Shift Right Arithmetic Immediate
11278 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11279   match(Set dst (RShiftL src1 src2));
11280 
11281   ins_cost(INSN_COST);
11282   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11283 
11284   ins_encode %{
11285     __ asr(as_Register($dst$$reg),
11286            as_Register($src1$$reg),
11287            $src2$$constant & 0x3f);
11288   %}
11289 
11290   ins_pipe(ialu_reg_shift);
11291 %}
11292 
11293 // BEGIN This section of the file is automatically generated. Do not edit --------------
11294 // This section is generated from aarch64_ad.m4
11295 
11296 
11297 // This pattern is automatically generated from aarch64_ad.m4
11298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11299 instruct regL_not_reg(iRegLNoSp dst,
11300                          iRegL src1, immL_M1 m1,
11301                          rFlagsReg cr) %{
11302   match(Set dst (XorL src1 m1));
11303   ins_cost(INSN_COST);
11304   format %{ "eon  $dst, $src1, zr" %}
11305 
11306   ins_encode %{
11307     __ eon(as_Register($dst$$reg),
11308               as_Register($src1$$reg),
11309               zr,
11310               Assembler::LSL, 0);
11311   %}
11312 
11313   ins_pipe(ialu_reg);
11314 %}
11315 
11316 // This pattern is automatically generated from aarch64_ad.m4
11317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11318 instruct regI_not_reg(iRegINoSp dst,
11319                          iRegIorL2I src1, immI_M1 m1,
11320                          rFlagsReg cr) %{
11321   match(Set dst (XorI src1 m1));
11322   ins_cost(INSN_COST);
11323   format %{ "eonw  $dst, $src1, zr" %}
11324 
11325   ins_encode %{
11326     __ eonw(as_Register($dst$$reg),
11327               as_Register($src1$$reg),
11328               zr,
11329               Assembler::LSL, 0);
11330   %}
11331 
11332   ins_pipe(ialu_reg);
11333 %}
11334 
11335 // This pattern is automatically generated from aarch64_ad.m4
11336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11337 instruct AndI_reg_not_reg(iRegINoSp dst,
11338                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11339   match(Set dst (AndI src1 (XorI src2 m1)));
11340   ins_cost(INSN_COST);
11341   format %{ "bicw  $dst, $src1, $src2" %}
11342 
11343   ins_encode %{
11344     __ bicw(as_Register($dst$$reg),
11345               as_Register($src1$$reg),
11346               as_Register($src2$$reg),
11347               Assembler::LSL, 0);
11348   %}
11349 
11350   ins_pipe(ialu_reg_reg);
11351 %}
11352 
11353 // This pattern is automatically generated from aarch64_ad.m4
11354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11355 instruct AndL_reg_not_reg(iRegLNoSp dst,
11356                          iRegL src1, iRegL src2, immL_M1 m1) %{
11357   match(Set dst (AndL src1 (XorL src2 m1)));
11358   ins_cost(INSN_COST);
11359   format %{ "bic  $dst, $src1, $src2" %}
11360 
11361   ins_encode %{
11362     __ bic(as_Register($dst$$reg),
11363               as_Register($src1$$reg),
11364               as_Register($src2$$reg),
11365               Assembler::LSL, 0);
11366   %}
11367 
11368   ins_pipe(ialu_reg_reg);
11369 %}
11370 
11371 // This pattern is automatically generated from aarch64_ad.m4
11372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11373 instruct OrI_reg_not_reg(iRegINoSp dst,
11374                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11375   match(Set dst (OrI src1 (XorI src2 m1)));
11376   ins_cost(INSN_COST);
11377   format %{ "ornw  $dst, $src1, $src2" %}
11378 
11379   ins_encode %{
11380     __ ornw(as_Register($dst$$reg),
11381               as_Register($src1$$reg),
11382               as_Register($src2$$reg),
11383               Assembler::LSL, 0);
11384   %}
11385 
11386   ins_pipe(ialu_reg_reg);
11387 %}
11388 
11389 // This pattern is automatically generated from aarch64_ad.m4
11390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11391 instruct OrL_reg_not_reg(iRegLNoSp dst,
11392                          iRegL src1, iRegL src2, immL_M1 m1) %{
11393   match(Set dst (OrL src1 (XorL src2 m1)));
11394   ins_cost(INSN_COST);
11395   format %{ "orn  $dst, $src1, $src2" %}
11396 
11397   ins_encode %{
11398     __ orn(as_Register($dst$$reg),
11399               as_Register($src1$$reg),
11400               as_Register($src2$$reg),
11401               Assembler::LSL, 0);
11402   %}
11403 
11404   ins_pipe(ialu_reg_reg);
11405 %}
11406 
11407 // This pattern is automatically generated from aarch64_ad.m4
11408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11409 instruct XorI_reg_not_reg(iRegINoSp dst,
11410                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11411   match(Set dst (XorI m1 (XorI src2 src1)));
11412   ins_cost(INSN_COST);
11413   format %{ "eonw  $dst, $src1, $src2" %}
11414 
11415   ins_encode %{
11416     __ eonw(as_Register($dst$$reg),
11417               as_Register($src1$$reg),
11418               as_Register($src2$$reg),
11419               Assembler::LSL, 0);
11420   %}
11421 
11422   ins_pipe(ialu_reg_reg);
11423 %}
11424 
11425 // This pattern is automatically generated from aarch64_ad.m4
11426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11427 instruct XorL_reg_not_reg(iRegLNoSp dst,
11428                          iRegL src1, iRegL src2, immL_M1 m1) %{
11429   match(Set dst (XorL m1 (XorL src2 src1)));
11430   ins_cost(INSN_COST);
11431   format %{ "eon  $dst, $src1, $src2" %}
11432 
11433   ins_encode %{
11434     __ eon(as_Register($dst$$reg),
11435               as_Register($src1$$reg),
11436               as_Register($src2$$reg),
11437               Assembler::LSL, 0);
11438   %}
11439 
11440   ins_pipe(ialu_reg_reg);
11441 %}
11442 
11443 // This pattern is automatically generated from aarch64_ad.m4
11444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11445 // val & (-1 ^ (val >>> shift)) ==> bicw
11446 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11447                          iRegIorL2I src1, iRegIorL2I src2,
11448                          immI src3, immI_M1 src4) %{
11449   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11450   ins_cost(1.9 * INSN_COST);
11451   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11452 
11453   ins_encode %{
11454     __ bicw(as_Register($dst$$reg),
11455               as_Register($src1$$reg),
11456               as_Register($src2$$reg),
11457               Assembler::LSR,
11458               $src3$$constant & 0x1f);
11459   %}
11460 
11461   ins_pipe(ialu_reg_reg_shift);
11462 %}
11463 
11464 // This pattern is automatically generated from aarch64_ad.m4
11465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11466 // val & (-1 ^ (val >>> shift)) ==> bic
11467 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11468                          iRegL src1, iRegL src2,
11469                          immI src3, immL_M1 src4) %{
11470   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11471   ins_cost(1.9 * INSN_COST);
11472   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11473 
11474   ins_encode %{
11475     __ bic(as_Register($dst$$reg),
11476               as_Register($src1$$reg),
11477               as_Register($src2$$reg),
11478               Assembler::LSR,
11479               $src3$$constant & 0x3f);
11480   %}
11481 
11482   ins_pipe(ialu_reg_reg_shift);
11483 %}
11484 
11485 // This pattern is automatically generated from aarch64_ad.m4
11486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11487 // val & (-1 ^ (val >> shift)) ==> bicw
11488 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11489                          iRegIorL2I src1, iRegIorL2I src2,
11490                          immI src3, immI_M1 src4) %{
11491   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11492   ins_cost(1.9 * INSN_COST);
11493   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11494 
11495   ins_encode %{
11496     __ bicw(as_Register($dst$$reg),
11497               as_Register($src1$$reg),
11498               as_Register($src2$$reg),
11499               Assembler::ASR,
11500               $src3$$constant & 0x1f);
11501   %}
11502 
11503   ins_pipe(ialu_reg_reg_shift);
11504 %}
11505 
11506 // This pattern is automatically generated from aarch64_ad.m4
11507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11508 // val & (-1 ^ (val >> shift)) ==> bic
11509 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11510                          iRegL src1, iRegL src2,
11511                          immI src3, immL_M1 src4) %{
11512   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11513   ins_cost(1.9 * INSN_COST);
11514   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11515 
11516   ins_encode %{
11517     __ bic(as_Register($dst$$reg),
11518               as_Register($src1$$reg),
11519               as_Register($src2$$reg),
11520               Assembler::ASR,
11521               $src3$$constant & 0x3f);
11522   %}
11523 
11524   ins_pipe(ialu_reg_reg_shift);
11525 %}
11526 
11527 // This pattern is automatically generated from aarch64_ad.m4
11528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11529 // val & (-1 ^ (val ror shift)) ==> bicw
11530 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11531                          iRegIorL2I src1, iRegIorL2I src2,
11532                          immI src3, immI_M1 src4) %{
11533   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11534   ins_cost(1.9 * INSN_COST);
11535   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11536 
11537   ins_encode %{
11538     __ bicw(as_Register($dst$$reg),
11539               as_Register($src1$$reg),
11540               as_Register($src2$$reg),
11541               Assembler::ROR,
11542               $src3$$constant & 0x1f);
11543   %}
11544 
11545   ins_pipe(ialu_reg_reg_shift);
11546 %}
11547 
11548 // This pattern is automatically generated from aarch64_ad.m4
11549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11550 // val & (-1 ^ (val ror shift)) ==> bic
11551 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11552                          iRegL src1, iRegL src2,
11553                          immI src3, immL_M1 src4) %{
11554   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11555   ins_cost(1.9 * INSN_COST);
11556   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11557 
11558   ins_encode %{
11559     __ bic(as_Register($dst$$reg),
11560               as_Register($src1$$reg),
11561               as_Register($src2$$reg),
11562               Assembler::ROR,
11563               $src3$$constant & 0x3f);
11564   %}
11565 
11566   ins_pipe(ialu_reg_reg_shift);
11567 %}
11568 
11569 // This pattern is automatically generated from aarch64_ad.m4
11570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11571 // val & (-1 ^ (val << shift)) ==> bicw
11572 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11573                          iRegIorL2I src1, iRegIorL2I src2,
11574                          immI src3, immI_M1 src4) %{
11575   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11576   ins_cost(1.9 * INSN_COST);
11577   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11578 
11579   ins_encode %{
11580     __ bicw(as_Register($dst$$reg),
11581               as_Register($src1$$reg),
11582               as_Register($src2$$reg),
11583               Assembler::LSL,
11584               $src3$$constant & 0x1f);
11585   %}
11586 
11587   ins_pipe(ialu_reg_reg_shift);
11588 %}
11589 
11590 // This pattern is automatically generated from aarch64_ad.m4
11591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11592 // val & (-1 ^ (val << shift)) ==> bic
11593 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11594                          iRegL src1, iRegL src2,
11595                          immI src3, immL_M1 src4) %{
11596   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11597   ins_cost(1.9 * INSN_COST);
11598   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11599 
11600   ins_encode %{
11601     __ bic(as_Register($dst$$reg),
11602               as_Register($src1$$reg),
11603               as_Register($src2$$reg),
11604               Assembler::LSL,
11605               $src3$$constant & 0x3f);
11606   %}
11607 
11608   ins_pipe(ialu_reg_reg_shift);
11609 %}
11610 
11611 // This pattern is automatically generated from aarch64_ad.m4
11612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11613 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11614 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11615                          iRegIorL2I src1, iRegIorL2I src2,
11616                          immI src3, immI_M1 src4) %{
11617   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11618   ins_cost(1.9 * INSN_COST);
11619   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11620 
11621   ins_encode %{
11622     __ eonw(as_Register($dst$$reg),
11623               as_Register($src1$$reg),
11624               as_Register($src2$$reg),
11625               Assembler::LSR,
11626               $src3$$constant & 0x1f);
11627   %}
11628 
11629   ins_pipe(ialu_reg_reg_shift);
11630 %}
11631 
11632 // This pattern is automatically generated from aarch64_ad.m4
11633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11634 // val ^ (-1 ^ (val >>> shift)) ==> eon
11635 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11636                          iRegL src1, iRegL src2,
11637                          immI src3, immL_M1 src4) %{
11638   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11639   ins_cost(1.9 * INSN_COST);
11640   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11641 
11642   ins_encode %{
11643     __ eon(as_Register($dst$$reg),
11644               as_Register($src1$$reg),
11645               as_Register($src2$$reg),
11646               Assembler::LSR,
11647               $src3$$constant & 0x3f);
11648   %}
11649 
11650   ins_pipe(ialu_reg_reg_shift);
11651 %}
11652 
11653 // This pattern is automatically generated from aarch64_ad.m4
11654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11655 // val ^ (-1 ^ (val >> shift)) ==> eonw
11656 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11657                          iRegIorL2I src1, iRegIorL2I src2,
11658                          immI src3, immI_M1 src4) %{
11659   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11660   ins_cost(1.9 * INSN_COST);
11661   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11662 
11663   ins_encode %{
11664     __ eonw(as_Register($dst$$reg),
11665               as_Register($src1$$reg),
11666               as_Register($src2$$reg),
11667               Assembler::ASR,
11668               $src3$$constant & 0x1f);
11669   %}
11670 
11671   ins_pipe(ialu_reg_reg_shift);
11672 %}
11673 
11674 // This pattern is automatically generated from aarch64_ad.m4
11675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11676 // val ^ (-1 ^ (val >> shift)) ==> eon
11677 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11678                          iRegL src1, iRegL src2,
11679                          immI src3, immL_M1 src4) %{
11680   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11681   ins_cost(1.9 * INSN_COST);
11682   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11683 
11684   ins_encode %{
11685     __ eon(as_Register($dst$$reg),
11686               as_Register($src1$$reg),
11687               as_Register($src2$$reg),
11688               Assembler::ASR,
11689               $src3$$constant & 0x3f);
11690   %}
11691 
11692   ins_pipe(ialu_reg_reg_shift);
11693 %}
11694 
11695 // This pattern is automatically generated from aarch64_ad.m4
11696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11697 // val ^ (-1 ^ (val ror shift)) ==> eonw
11698 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11699                          iRegIorL2I src1, iRegIorL2I src2,
11700                          immI src3, immI_M1 src4) %{
11701   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11702   ins_cost(1.9 * INSN_COST);
11703   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11704 
11705   ins_encode %{
11706     __ eonw(as_Register($dst$$reg),
11707               as_Register($src1$$reg),
11708               as_Register($src2$$reg),
11709               Assembler::ROR,
11710               $src3$$constant & 0x1f);
11711   %}
11712 
11713   ins_pipe(ialu_reg_reg_shift);
11714 %}
11715 
11716 // This pattern is automatically generated from aarch64_ad.m4
11717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11718 // val ^ (-1 ^ (val ror shift)) ==> eon
11719 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11720                          iRegL src1, iRegL src2,
11721                          immI src3, immL_M1 src4) %{
11722   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11723   ins_cost(1.9 * INSN_COST);
11724   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11725 
11726   ins_encode %{
11727     __ eon(as_Register($dst$$reg),
11728               as_Register($src1$$reg),
11729               as_Register($src2$$reg),
11730               Assembler::ROR,
11731               $src3$$constant & 0x3f);
11732   %}
11733 
11734   ins_pipe(ialu_reg_reg_shift);
11735 %}
11736 
11737 // This pattern is automatically generated from aarch64_ad.m4
11738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11739 // val ^ (-1 ^ (val << shift)) ==> eonw
11740 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11741                          iRegIorL2I src1, iRegIorL2I src2,
11742                          immI src3, immI_M1 src4) %{
11743   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11744   ins_cost(1.9 * INSN_COST);
11745   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11746 
11747   ins_encode %{
11748     __ eonw(as_Register($dst$$reg),
11749               as_Register($src1$$reg),
11750               as_Register($src2$$reg),
11751               Assembler::LSL,
11752               $src3$$constant & 0x1f);
11753   %}
11754 
11755   ins_pipe(ialu_reg_reg_shift);
11756 %}
11757 
11758 // This pattern is automatically generated from aarch64_ad.m4
11759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11760 // val ^ (-1 ^ (val << shift)) ==> eon
11761 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11762                          iRegL src1, iRegL src2,
11763                          immI src3, immL_M1 src4) %{
11764   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11765   ins_cost(1.9 * INSN_COST);
11766   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11767 
11768   ins_encode %{
11769     __ eon(as_Register($dst$$reg),
11770               as_Register($src1$$reg),
11771               as_Register($src2$$reg),
11772               Assembler::LSL,
11773               $src3$$constant & 0x3f);
11774   %}
11775 
11776   ins_pipe(ialu_reg_reg_shift);
11777 %}
11778 
11779 // This pattern is automatically generated from aarch64_ad.m4
11780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11781 // val | (-1 ^ (val >>> shift)) ==> ornw
11782 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11783                          iRegIorL2I src1, iRegIorL2I src2,
11784                          immI src3, immI_M1 src4) %{
11785   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11786   ins_cost(1.9 * INSN_COST);
11787   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11788 
11789   ins_encode %{
11790     __ ornw(as_Register($dst$$reg),
11791               as_Register($src1$$reg),
11792               as_Register($src2$$reg),
11793               Assembler::LSR,
11794               $src3$$constant & 0x1f);
11795   %}
11796 
11797   ins_pipe(ialu_reg_reg_shift);
11798 %}
11799 
11800 // This pattern is automatically generated from aarch64_ad.m4
11801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11802 // val | (-1 ^ (val >>> shift)) ==> orn
11803 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11804                          iRegL src1, iRegL src2,
11805                          immI src3, immL_M1 src4) %{
11806   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11807   ins_cost(1.9 * INSN_COST);
11808   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11809 
11810   ins_encode %{
11811     __ orn(as_Register($dst$$reg),
11812               as_Register($src1$$reg),
11813               as_Register($src2$$reg),
11814               Assembler::LSR,
11815               $src3$$constant & 0x3f);
11816   %}
11817 
11818   ins_pipe(ialu_reg_reg_shift);
11819 %}
11820 
11821 // This pattern is automatically generated from aarch64_ad.m4
11822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11823 // val | (-1 ^ (val >> shift)) ==> ornw
11824 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11825                          iRegIorL2I src1, iRegIorL2I src2,
11826                          immI src3, immI_M1 src4) %{
11827   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11828   ins_cost(1.9 * INSN_COST);
11829   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11830 
11831   ins_encode %{
11832     __ ornw(as_Register($dst$$reg),
11833               as_Register($src1$$reg),
11834               as_Register($src2$$reg),
11835               Assembler::ASR,
11836               $src3$$constant & 0x1f);
11837   %}
11838 
11839   ins_pipe(ialu_reg_reg_shift);
11840 %}
11841 
11842 // This pattern is automatically generated from aarch64_ad.m4
11843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11844 // val | (-1 ^ (val >> shift)) ==> orn
11845 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11846                          iRegL src1, iRegL src2,
11847                          immI src3, immL_M1 src4) %{
11848   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11849   ins_cost(1.9 * INSN_COST);
11850   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11851 
11852   ins_encode %{
11853     __ orn(as_Register($dst$$reg),
11854               as_Register($src1$$reg),
11855               as_Register($src2$$reg),
11856               Assembler::ASR,
11857               $src3$$constant & 0x3f);
11858   %}
11859 
11860   ins_pipe(ialu_reg_reg_shift);
11861 %}
11862 
11863 // This pattern is automatically generated from aarch64_ad.m4
11864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11865 // val | (-1 ^ (val ror shift)) ==> ornw
11866 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11867                          iRegIorL2I src1, iRegIorL2I src2,
11868                          immI src3, immI_M1 src4) %{
11869   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11870   ins_cost(1.9 * INSN_COST);
11871   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11872 
11873   ins_encode %{
11874     __ ornw(as_Register($dst$$reg),
11875               as_Register($src1$$reg),
11876               as_Register($src2$$reg),
11877               Assembler::ROR,
11878               $src3$$constant & 0x1f);
11879   %}
11880 
11881   ins_pipe(ialu_reg_reg_shift);
11882 %}
11883 
11884 // This pattern is automatically generated from aarch64_ad.m4
11885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11886 // val | (-1 ^ (val ror shift)) ==> orn
11887 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11888                          iRegL src1, iRegL src2,
11889                          immI src3, immL_M1 src4) %{
11890   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11891   ins_cost(1.9 * INSN_COST);
11892   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11893 
11894   ins_encode %{
11895     __ orn(as_Register($dst$$reg),
11896               as_Register($src1$$reg),
11897               as_Register($src2$$reg),
11898               Assembler::ROR,
11899               $src3$$constant & 0x3f);
11900   %}
11901 
11902   ins_pipe(ialu_reg_reg_shift);
11903 %}
11904 
11905 // This pattern is automatically generated from aarch64_ad.m4
11906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11907 // val | (-1 ^ (val << shift)) ==> ornw
11908 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11909                          iRegIorL2I src1, iRegIorL2I src2,
11910                          immI src3, immI_M1 src4) %{
11911   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11912   ins_cost(1.9 * INSN_COST);
11913   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11914 
11915   ins_encode %{
11916     __ ornw(as_Register($dst$$reg),
11917               as_Register($src1$$reg),
11918               as_Register($src2$$reg),
11919               Assembler::LSL,
11920               $src3$$constant & 0x1f);
11921   %}
11922 
11923   ins_pipe(ialu_reg_reg_shift);
11924 %}
11925 
11926 // This pattern is automatically generated from aarch64_ad.m4
11927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11928 // val | (-1 ^ (val << shift)) ==> orn
11929 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11930                          iRegL src1, iRegL src2,
11931                          immI src3, immL_M1 src4) %{
11932   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11933   ins_cost(1.9 * INSN_COST);
11934   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11935 
11936   ins_encode %{
11937     __ orn(as_Register($dst$$reg),
11938               as_Register($src1$$reg),
11939               as_Register($src2$$reg),
11940               Assembler::LSL,
11941               $src3$$constant & 0x3f);
11942   %}
11943 
11944   ins_pipe(ialu_reg_reg_shift);
11945 %}
11946 
11947 // This pattern is automatically generated from aarch64_ad.m4
11948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11949 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11950                          iRegIorL2I src1, iRegIorL2I src2,
11951                          immI src3) %{
11952   match(Set dst (AndI src1 (URShiftI src2 src3)));
11953 
11954   ins_cost(1.9 * INSN_COST);
11955   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11956 
11957   ins_encode %{
11958     __ andw(as_Register($dst$$reg),
11959               as_Register($src1$$reg),
11960               as_Register($src2$$reg),
11961               Assembler::LSR,
11962               $src3$$constant & 0x1f);
11963   %}
11964 
11965   ins_pipe(ialu_reg_reg_shift);
11966 %}
11967 
11968 // This pattern is automatically generated from aarch64_ad.m4
11969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11970 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11971                          iRegL src1, iRegL src2,
11972                          immI src3) %{
11973   match(Set dst (AndL src1 (URShiftL src2 src3)));
11974 
11975   ins_cost(1.9 * INSN_COST);
11976   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11977 
11978   ins_encode %{
11979     __ andr(as_Register($dst$$reg),
11980               as_Register($src1$$reg),
11981               as_Register($src2$$reg),
11982               Assembler::LSR,
11983               $src3$$constant & 0x3f);
11984   %}
11985 
11986   ins_pipe(ialu_reg_reg_shift);
11987 %}
11988 
11989 // This pattern is automatically generated from aarch64_ad.m4
11990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11991 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11992                          iRegIorL2I src1, iRegIorL2I src2,
11993                          immI src3) %{
11994   match(Set dst (AndI src1 (RShiftI src2 src3)));
11995 
11996   ins_cost(1.9 * INSN_COST);
11997   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11998 
11999   ins_encode %{
12000     __ andw(as_Register($dst$$reg),
12001               as_Register($src1$$reg),
12002               as_Register($src2$$reg),
12003               Assembler::ASR,
12004               $src3$$constant & 0x1f);
12005   %}
12006 
12007   ins_pipe(ialu_reg_reg_shift);
12008 %}
12009 
12010 // This pattern is automatically generated from aarch64_ad.m4
12011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12012 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12013                          iRegL src1, iRegL src2,
12014                          immI src3) %{
12015   match(Set dst (AndL src1 (RShiftL src2 src3)));
12016 
12017   ins_cost(1.9 * INSN_COST);
12018   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12019 
12020   ins_encode %{
12021     __ andr(as_Register($dst$$reg),
12022               as_Register($src1$$reg),
12023               as_Register($src2$$reg),
12024               Assembler::ASR,
12025               $src3$$constant & 0x3f);
12026   %}
12027 
12028   ins_pipe(ialu_reg_reg_shift);
12029 %}
12030 
12031 // This pattern is automatically generated from aarch64_ad.m4
12032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12033 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12034                          iRegIorL2I src1, iRegIorL2I src2,
12035                          immI src3) %{
12036   match(Set dst (AndI src1 (LShiftI src2 src3)));
12037 
12038   ins_cost(1.9 * INSN_COST);
12039   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12040 
12041   ins_encode %{
12042     __ andw(as_Register($dst$$reg),
12043               as_Register($src1$$reg),
12044               as_Register($src2$$reg),
12045               Assembler::LSL,
12046               $src3$$constant & 0x1f);
12047   %}
12048 
12049   ins_pipe(ialu_reg_reg_shift);
12050 %}
12051 
12052 // This pattern is automatically generated from aarch64_ad.m4
12053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12054 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12055                          iRegL src1, iRegL src2,
12056                          immI src3) %{
12057   match(Set dst (AndL src1 (LShiftL src2 src3)));
12058 
12059   ins_cost(1.9 * INSN_COST);
12060   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12061 
12062   ins_encode %{
12063     __ andr(as_Register($dst$$reg),
12064               as_Register($src1$$reg),
12065               as_Register($src2$$reg),
12066               Assembler::LSL,
12067               $src3$$constant & 0x3f);
12068   %}
12069 
12070   ins_pipe(ialu_reg_reg_shift);
12071 %}
12072 
12073 // This pattern is automatically generated from aarch64_ad.m4
12074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12075 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12076                          iRegIorL2I src1, iRegIorL2I src2,
12077                          immI src3) %{
12078   match(Set dst (AndI src1 (RotateRight src2 src3)));
12079 
12080   ins_cost(1.9 * INSN_COST);
12081   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12082 
12083   ins_encode %{
12084     __ andw(as_Register($dst$$reg),
12085               as_Register($src1$$reg),
12086               as_Register($src2$$reg),
12087               Assembler::ROR,
12088               $src3$$constant & 0x1f);
12089   %}
12090 
12091   ins_pipe(ialu_reg_reg_shift);
12092 %}
12093 
12094 // This pattern is automatically generated from aarch64_ad.m4
12095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12096 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12097                          iRegL src1, iRegL src2,
12098                          immI src3) %{
12099   match(Set dst (AndL src1 (RotateRight src2 src3)));
12100 
12101   ins_cost(1.9 * INSN_COST);
12102   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12103 
12104   ins_encode %{
12105     __ andr(as_Register($dst$$reg),
12106               as_Register($src1$$reg),
12107               as_Register($src2$$reg),
12108               Assembler::ROR,
12109               $src3$$constant & 0x3f);
12110   %}
12111 
12112   ins_pipe(ialu_reg_reg_shift);
12113 %}
12114 
12115 // This pattern is automatically generated from aarch64_ad.m4
12116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12117 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12118                          iRegIorL2I src1, iRegIorL2I src2,
12119                          immI src3) %{
12120   match(Set dst (XorI src1 (URShiftI src2 src3)));
12121 
12122   ins_cost(1.9 * INSN_COST);
12123   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12124 
12125   ins_encode %{
12126     __ eorw(as_Register($dst$$reg),
12127               as_Register($src1$$reg),
12128               as_Register($src2$$reg),
12129               Assembler::LSR,
12130               $src3$$constant & 0x1f);
12131   %}
12132 
12133   ins_pipe(ialu_reg_reg_shift);
12134 %}
12135 
12136 // This pattern is automatically generated from aarch64_ad.m4
12137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12138 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12139                          iRegL src1, iRegL src2,
12140                          immI src3) %{
12141   match(Set dst (XorL src1 (URShiftL src2 src3)));
12142 
12143   ins_cost(1.9 * INSN_COST);
12144   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12145 
12146   ins_encode %{
12147     __ eor(as_Register($dst$$reg),
12148               as_Register($src1$$reg),
12149               as_Register($src2$$reg),
12150               Assembler::LSR,
12151               $src3$$constant & 0x3f);
12152   %}
12153 
12154   ins_pipe(ialu_reg_reg_shift);
12155 %}
12156 
12157 // This pattern is automatically generated from aarch64_ad.m4
12158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12159 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12160                          iRegIorL2I src1, iRegIorL2I src2,
12161                          immI src3) %{
12162   match(Set dst (XorI src1 (RShiftI src2 src3)));
12163 
12164   ins_cost(1.9 * INSN_COST);
12165   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12166 
12167   ins_encode %{
12168     __ eorw(as_Register($dst$$reg),
12169               as_Register($src1$$reg),
12170               as_Register($src2$$reg),
12171               Assembler::ASR,
12172               $src3$$constant & 0x1f);
12173   %}
12174 
12175   ins_pipe(ialu_reg_reg_shift);
12176 %}
12177 
12178 // This pattern is automatically generated from aarch64_ad.m4
12179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12180 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12181                          iRegL src1, iRegL src2,
12182                          immI src3) %{
12183   match(Set dst (XorL src1 (RShiftL src2 src3)));
12184 
12185   ins_cost(1.9 * INSN_COST);
12186   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12187 
12188   ins_encode %{
12189     __ eor(as_Register($dst$$reg),
12190               as_Register($src1$$reg),
12191               as_Register($src2$$reg),
12192               Assembler::ASR,
12193               $src3$$constant & 0x3f);
12194   %}
12195 
12196   ins_pipe(ialu_reg_reg_shift);
12197 %}
12198 
12199 // This pattern is automatically generated from aarch64_ad.m4
12200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12201 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12202                          iRegIorL2I src1, iRegIorL2I src2,
12203                          immI src3) %{
12204   match(Set dst (XorI src1 (LShiftI src2 src3)));
12205 
12206   ins_cost(1.9 * INSN_COST);
12207   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12208 
12209   ins_encode %{
12210     __ eorw(as_Register($dst$$reg),
12211               as_Register($src1$$reg),
12212               as_Register($src2$$reg),
12213               Assembler::LSL,
12214               $src3$$constant & 0x1f);
12215   %}
12216 
12217   ins_pipe(ialu_reg_reg_shift);
12218 %}
12219 
12220 // This pattern is automatically generated from aarch64_ad.m4
12221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12222 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12223                          iRegL src1, iRegL src2,
12224                          immI src3) %{
12225   match(Set dst (XorL src1 (LShiftL src2 src3)));
12226 
12227   ins_cost(1.9 * INSN_COST);
12228   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12229 
12230   ins_encode %{
12231     __ eor(as_Register($dst$$reg),
12232               as_Register($src1$$reg),
12233               as_Register($src2$$reg),
12234               Assembler::LSL,
12235               $src3$$constant & 0x3f);
12236   %}
12237 
12238   ins_pipe(ialu_reg_reg_shift);
12239 %}
12240 
12241 // This pattern is automatically generated from aarch64_ad.m4
12242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12243 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12244                          iRegIorL2I src1, iRegIorL2I src2,
12245                          immI src3) %{
12246   match(Set dst (XorI src1 (RotateRight src2 src3)));
12247 
12248   ins_cost(1.9 * INSN_COST);
12249   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12250 
12251   ins_encode %{
12252     __ eorw(as_Register($dst$$reg),
12253               as_Register($src1$$reg),
12254               as_Register($src2$$reg),
12255               Assembler::ROR,
12256               $src3$$constant & 0x1f);
12257   %}
12258 
12259   ins_pipe(ialu_reg_reg_shift);
12260 %}
12261 
12262 // This pattern is automatically generated from aarch64_ad.m4
12263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12264 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12265                          iRegL src1, iRegL src2,
12266                          immI src3) %{
12267   match(Set dst (XorL src1 (RotateRight src2 src3)));
12268 
12269   ins_cost(1.9 * INSN_COST);
12270   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12271 
12272   ins_encode %{
12273     __ eor(as_Register($dst$$reg),
12274               as_Register($src1$$reg),
12275               as_Register($src2$$reg),
12276               Assembler::ROR,
12277               $src3$$constant & 0x3f);
12278   %}
12279 
12280   ins_pipe(ialu_reg_reg_shift);
12281 %}
12282 
12283 // This pattern is automatically generated from aarch64_ad.m4
12284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12285 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12286                          iRegIorL2I src1, iRegIorL2I src2,
12287                          immI src3) %{
12288   match(Set dst (OrI src1 (URShiftI src2 src3)));
12289 
12290   ins_cost(1.9 * INSN_COST);
12291   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12292 
12293   ins_encode %{
12294     __ orrw(as_Register($dst$$reg),
12295               as_Register($src1$$reg),
12296               as_Register($src2$$reg),
12297               Assembler::LSR,
12298               $src3$$constant & 0x1f);
12299   %}
12300 
12301   ins_pipe(ialu_reg_reg_shift);
12302 %}
12303 
12304 // This pattern is automatically generated from aarch64_ad.m4
12305 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12306 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12307                          iRegL src1, iRegL src2,
12308                          immI src3) %{
12309   match(Set dst (OrL src1 (URShiftL src2 src3)));
12310 
12311   ins_cost(1.9 * INSN_COST);
12312   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12313 
12314   ins_encode %{
12315     __ orr(as_Register($dst$$reg),
12316               as_Register($src1$$reg),
12317               as_Register($src2$$reg),
12318               Assembler::LSR,
12319               $src3$$constant & 0x3f);
12320   %}
12321 
12322   ins_pipe(ialu_reg_reg_shift);
12323 %}
12324 
12325 // This pattern is automatically generated from aarch64_ad.m4
12326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12327 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12328                          iRegIorL2I src1, iRegIorL2I src2,
12329                          immI src3) %{
12330   match(Set dst (OrI src1 (RShiftI src2 src3)));
12331 
12332   ins_cost(1.9 * INSN_COST);
12333   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12334 
12335   ins_encode %{
12336     __ orrw(as_Register($dst$$reg),
12337               as_Register($src1$$reg),
12338               as_Register($src2$$reg),
12339               Assembler::ASR,
12340               $src3$$constant & 0x1f);
12341   %}
12342 
12343   ins_pipe(ialu_reg_reg_shift);
12344 %}
12345 
12346 // This pattern is automatically generated from aarch64_ad.m4
12347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12348 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12349                          iRegL src1, iRegL src2,
12350                          immI src3) %{
12351   match(Set dst (OrL src1 (RShiftL src2 src3)));
12352 
12353   ins_cost(1.9 * INSN_COST);
12354   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12355 
12356   ins_encode %{
12357     __ orr(as_Register($dst$$reg),
12358               as_Register($src1$$reg),
12359               as_Register($src2$$reg),
12360               Assembler::ASR,
12361               $src3$$constant & 0x3f);
12362   %}
12363 
12364   ins_pipe(ialu_reg_reg_shift);
12365 %}
12366 
12367 // This pattern is automatically generated from aarch64_ad.m4
12368 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12369 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12370                          iRegIorL2I src1, iRegIorL2I src2,
12371                          immI src3) %{
12372   match(Set dst (OrI src1 (LShiftI src2 src3)));
12373 
12374   ins_cost(1.9 * INSN_COST);
12375   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12376 
12377   ins_encode %{
12378     __ orrw(as_Register($dst$$reg),
12379               as_Register($src1$$reg),
12380               as_Register($src2$$reg),
12381               Assembler::LSL,
12382               $src3$$constant & 0x1f);
12383   %}
12384 
12385   ins_pipe(ialu_reg_reg_shift);
12386 %}
12387 
12388 // This pattern is automatically generated from aarch64_ad.m4
12389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12390 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12391                          iRegL src1, iRegL src2,
12392                          immI src3) %{
12393   match(Set dst (OrL src1 (LShiftL src2 src3)));
12394 
12395   ins_cost(1.9 * INSN_COST);
12396   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12397 
12398   ins_encode %{
12399     __ orr(as_Register($dst$$reg),
12400               as_Register($src1$$reg),
12401               as_Register($src2$$reg),
12402               Assembler::LSL,
12403               $src3$$constant & 0x3f);
12404   %}
12405 
12406   ins_pipe(ialu_reg_reg_shift);
12407 %}
12408 
12409 // This pattern is automatically generated from aarch64_ad.m4
12410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12411 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12412                          iRegIorL2I src1, iRegIorL2I src2,
12413                          immI src3) %{
12414   match(Set dst (OrI src1 (RotateRight src2 src3)));
12415 
12416   ins_cost(1.9 * INSN_COST);
12417   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12418 
12419   ins_encode %{
12420     __ orrw(as_Register($dst$$reg),
12421               as_Register($src1$$reg),
12422               as_Register($src2$$reg),
12423               Assembler::ROR,
12424               $src3$$constant & 0x1f);
12425   %}
12426 
12427   ins_pipe(ialu_reg_reg_shift);
12428 %}
12429 
12430 // This pattern is automatically generated from aarch64_ad.m4
12431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12432 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12433                          iRegL src1, iRegL src2,
12434                          immI src3) %{
12435   match(Set dst (OrL src1 (RotateRight src2 src3)));
12436 
12437   ins_cost(1.9 * INSN_COST);
12438   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12439 
12440   ins_encode %{
12441     __ orr(as_Register($dst$$reg),
12442               as_Register($src1$$reg),
12443               as_Register($src2$$reg),
12444               Assembler::ROR,
12445               $src3$$constant & 0x3f);
12446   %}
12447 
12448   ins_pipe(ialu_reg_reg_shift);
12449 %}
12450 
12451 // This pattern is automatically generated from aarch64_ad.m4
12452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12453 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12454                          iRegIorL2I src1, iRegIorL2I src2,
12455                          immI src3) %{
12456   match(Set dst (AddI src1 (URShiftI src2 src3)));
12457 
12458   ins_cost(1.9 * INSN_COST);
12459   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12460 
12461   ins_encode %{
12462     __ addw(as_Register($dst$$reg),
12463               as_Register($src1$$reg),
12464               as_Register($src2$$reg),
12465               Assembler::LSR,
12466               $src3$$constant & 0x1f);
12467   %}
12468 
12469   ins_pipe(ialu_reg_reg_shift);
12470 %}
12471 
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12475                          iRegL src1, iRegL src2,
12476                          immI src3) %{
12477   match(Set dst (AddL src1 (URShiftL src2 src3)));
12478 
12479   ins_cost(1.9 * INSN_COST);
12480   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12481 
12482   ins_encode %{
12483     __ add(as_Register($dst$$reg),
12484               as_Register($src1$$reg),
12485               as_Register($src2$$reg),
12486               Assembler::LSR,
12487               $src3$$constant & 0x3f);
12488   %}
12489 
12490   ins_pipe(ialu_reg_reg_shift);
12491 %}
12492 
12493 // This pattern is automatically generated from aarch64_ad.m4
12494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12495 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12496                          iRegIorL2I src1, iRegIorL2I src2,
12497                          immI src3) %{
12498   match(Set dst (AddI src1 (RShiftI src2 src3)));
12499 
12500   ins_cost(1.9 * INSN_COST);
12501   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12502 
12503   ins_encode %{
12504     __ addw(as_Register($dst$$reg),
12505               as_Register($src1$$reg),
12506               as_Register($src2$$reg),
12507               Assembler::ASR,
12508               $src3$$constant & 0x1f);
12509   %}
12510 
12511   ins_pipe(ialu_reg_reg_shift);
12512 %}
12513 
12514 // This pattern is automatically generated from aarch64_ad.m4
12515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12516 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12517                          iRegL src1, iRegL src2,
12518                          immI src3) %{
12519   match(Set dst (AddL src1 (RShiftL src2 src3)));
12520 
12521   ins_cost(1.9 * INSN_COST);
12522   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12523 
12524   ins_encode %{
12525     __ add(as_Register($dst$$reg),
12526               as_Register($src1$$reg),
12527               as_Register($src2$$reg),
12528               Assembler::ASR,
12529               $src3$$constant & 0x3f);
12530   %}
12531 
12532   ins_pipe(ialu_reg_reg_shift);
12533 %}
12534 
12535 // This pattern is automatically generated from aarch64_ad.m4
12536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12537 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12538                          iRegIorL2I src1, iRegIorL2I src2,
12539                          immI src3) %{
12540   match(Set dst (AddI src1 (LShiftI src2 src3)));
12541 
12542   ins_cost(1.9 * INSN_COST);
12543   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12544 
12545   ins_encode %{
12546     __ addw(as_Register($dst$$reg),
12547               as_Register($src1$$reg),
12548               as_Register($src2$$reg),
12549               Assembler::LSL,
12550               $src3$$constant & 0x1f);
12551   %}
12552 
12553   ins_pipe(ialu_reg_reg_shift);
12554 %}
12555 
12556 // This pattern is automatically generated from aarch64_ad.m4
12557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12558 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12559                          iRegL src1, iRegL src2,
12560                          immI src3) %{
12561   match(Set dst (AddL src1 (LShiftL src2 src3)));
12562 
12563   ins_cost(1.9 * INSN_COST);
12564   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12565 
12566   ins_encode %{
12567     __ add(as_Register($dst$$reg),
12568               as_Register($src1$$reg),
12569               as_Register($src2$$reg),
12570               Assembler::LSL,
12571               $src3$$constant & 0x3f);
12572   %}
12573 
12574   ins_pipe(ialu_reg_reg_shift);
12575 %}
12576 
12577 // This pattern is automatically generated from aarch64_ad.m4
12578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12579 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12580                          iRegIorL2I src1, iRegIorL2I src2,
12581                          immI src3) %{
12582   match(Set dst (SubI src1 (URShiftI src2 src3)));
12583 
12584   ins_cost(1.9 * INSN_COST);
12585   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12586 
12587   ins_encode %{
12588     __ subw(as_Register($dst$$reg),
12589               as_Register($src1$$reg),
12590               as_Register($src2$$reg),
12591               Assembler::LSR,
12592               $src3$$constant & 0x1f);
12593   %}
12594 
12595   ins_pipe(ialu_reg_reg_shift);
12596 %}
12597 
12598 // This pattern is automatically generated from aarch64_ad.m4
12599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12600 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12601                          iRegL src1, iRegL src2,
12602                          immI src3) %{
12603   match(Set dst (SubL src1 (URShiftL src2 src3)));
12604 
12605   ins_cost(1.9 * INSN_COST);
12606   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12607 
12608   ins_encode %{
12609     __ sub(as_Register($dst$$reg),
12610               as_Register($src1$$reg),
12611               as_Register($src2$$reg),
12612               Assembler::LSR,
12613               $src3$$constant & 0x3f);
12614   %}
12615 
12616   ins_pipe(ialu_reg_reg_shift);
12617 %}
12618 
12619 // This pattern is automatically generated from aarch64_ad.m4
12620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12621 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12622                          iRegIorL2I src1, iRegIorL2I src2,
12623                          immI src3) %{
12624   match(Set dst (SubI src1 (RShiftI src2 src3)));
12625 
12626   ins_cost(1.9 * INSN_COST);
12627   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12628 
12629   ins_encode %{
12630     __ subw(as_Register($dst$$reg),
12631               as_Register($src1$$reg),
12632               as_Register($src2$$reg),
12633               Assembler::ASR,
12634               $src3$$constant & 0x1f);
12635   %}
12636 
12637   ins_pipe(ialu_reg_reg_shift);
12638 %}
12639 
12640 // This pattern is automatically generated from aarch64_ad.m4
12641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12642 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12643                          iRegL src1, iRegL src2,
12644                          immI src3) %{
12645   match(Set dst (SubL src1 (RShiftL src2 src3)));
12646 
12647   ins_cost(1.9 * INSN_COST);
12648   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12649 
12650   ins_encode %{
12651     __ sub(as_Register($dst$$reg),
12652               as_Register($src1$$reg),
12653               as_Register($src2$$reg),
12654               Assembler::ASR,
12655               $src3$$constant & 0x3f);
12656   %}
12657 
12658   ins_pipe(ialu_reg_reg_shift);
12659 %}
12660 
12661 // This pattern is automatically generated from aarch64_ad.m4
12662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12663 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12664                          iRegIorL2I src1, iRegIorL2I src2,
12665                          immI src3) %{
12666   match(Set dst (SubI src1 (LShiftI src2 src3)));
12667 
12668   ins_cost(1.9 * INSN_COST);
12669   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12670 
12671   ins_encode %{
12672     __ subw(as_Register($dst$$reg),
12673               as_Register($src1$$reg),
12674               as_Register($src2$$reg),
12675               Assembler::LSL,
12676               $src3$$constant & 0x1f);
12677   %}
12678 
12679   ins_pipe(ialu_reg_reg_shift);
12680 %}
12681 
12682 // This pattern is automatically generated from aarch64_ad.m4
12683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12684 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12685                          iRegL src1, iRegL src2,
12686                          immI src3) %{
12687   match(Set dst (SubL src1 (LShiftL src2 src3)));
12688 
12689   ins_cost(1.9 * INSN_COST);
12690   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12691 
12692   ins_encode %{
12693     __ sub(as_Register($dst$$reg),
12694               as_Register($src1$$reg),
12695               as_Register($src2$$reg),
12696               Assembler::LSL,
12697               $src3$$constant & 0x3f);
12698   %}
12699 
12700   ins_pipe(ialu_reg_reg_shift);
12701 %}
12702 
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705 
12706 // Shift Left followed by Shift Right.
12707 // This idiom is used by the compiler for the i2b bytecode etc.
12708 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12709 %{
12710   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12711   ins_cost(INSN_COST * 2);
12712   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12713   ins_encode %{
12714     int lshift = $lshift_count$$constant & 63;
12715     int rshift = $rshift_count$$constant & 63;
12716     int s = 63 - lshift;
12717     int r = (rshift - lshift) & 63;
12718     __ sbfm(as_Register($dst$$reg),
12719             as_Register($src$$reg),
12720             r, s);
12721   %}
12722 
12723   ins_pipe(ialu_reg_shift);
12724 %}
12725 
12726 // This pattern is automatically generated from aarch64_ad.m4
12727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12728 
12729 // Shift Left followed by Shift Right.
12730 // This idiom is used by the compiler for the i2b bytecode etc.
12731 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12732 %{
12733   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12734   ins_cost(INSN_COST * 2);
12735   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12736   ins_encode %{
12737     int lshift = $lshift_count$$constant & 31;
12738     int rshift = $rshift_count$$constant & 31;
12739     int s = 31 - lshift;
12740     int r = (rshift - lshift) & 31;
12741     __ sbfmw(as_Register($dst$$reg),
12742             as_Register($src$$reg),
12743             r, s);
12744   %}
12745 
12746   ins_pipe(ialu_reg_shift);
12747 %}
12748 
12749 // This pattern is automatically generated from aarch64_ad.m4
12750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12751 
12752 // Shift Left followed by Shift Right.
12753 // This idiom is used by the compiler for the i2b bytecode etc.
12754 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12755 %{
12756   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12757   ins_cost(INSN_COST * 2);
12758   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12759   ins_encode %{
12760     int lshift = $lshift_count$$constant & 63;
12761     int rshift = $rshift_count$$constant & 63;
12762     int s = 63 - lshift;
12763     int r = (rshift - lshift) & 63;
12764     __ ubfm(as_Register($dst$$reg),
12765             as_Register($src$$reg),
12766             r, s);
12767   %}
12768 
12769   ins_pipe(ialu_reg_shift);
12770 %}
12771 
12772 // This pattern is automatically generated from aarch64_ad.m4
12773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12774 
12775 // Shift Left followed by Shift Right.
12776 // This idiom is used by the compiler for the i2b bytecode etc.
12777 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12778 %{
12779   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12780   ins_cost(INSN_COST * 2);
12781   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12782   ins_encode %{
12783     int lshift = $lshift_count$$constant & 31;
12784     int rshift = $rshift_count$$constant & 31;
12785     int s = 31 - lshift;
12786     int r = (rshift - lshift) & 31;
12787     __ ubfmw(as_Register($dst$$reg),
12788             as_Register($src$$reg),
12789             r, s);
12790   %}
12791 
12792   ins_pipe(ialu_reg_shift);
12793 %}
12794 
12795 // Bitfield extract with shift & mask
12796 
12797 // This pattern is automatically generated from aarch64_ad.m4
12798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12799 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12800 %{
12801   match(Set dst (AndI (URShiftI src rshift) mask));
12802   // Make sure we are not going to exceed what ubfxw can do.
12803   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12804 
12805   ins_cost(INSN_COST);
12806   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12807   ins_encode %{
12808     int rshift = $rshift$$constant & 31;
12809     intptr_t mask = $mask$$constant;
12810     int width = exact_log2(mask+1);
12811     __ ubfxw(as_Register($dst$$reg),
12812             as_Register($src$$reg), rshift, width);
12813   %}
12814   ins_pipe(ialu_reg_shift);
12815 %}
12816 
12817 // This pattern is automatically generated from aarch64_ad.m4
12818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12819 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12820 %{
12821   match(Set dst (AndL (URShiftL src rshift) mask));
12822   // Make sure we are not going to exceed what ubfx can do.
12823   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12824 
12825   ins_cost(INSN_COST);
12826   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12827   ins_encode %{
12828     int rshift = $rshift$$constant & 63;
12829     intptr_t mask = $mask$$constant;
12830     int width = exact_log2_long(mask+1);
12831     __ ubfx(as_Register($dst$$reg),
12832             as_Register($src$$reg), rshift, width);
12833   %}
12834   ins_pipe(ialu_reg_shift);
12835 %}
12836 
12837 
12838 // This pattern is automatically generated from aarch64_ad.m4
12839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12840 
12841 // We can use ubfx when extending an And with a mask when we know mask
12842 // is positive.  We know that because immI_bitmask guarantees it.
12843 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12844 %{
12845   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12846   // Make sure we are not going to exceed what ubfxw can do.
12847   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12848 
12849   ins_cost(INSN_COST * 2);
12850   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12851   ins_encode %{
12852     int rshift = $rshift$$constant & 31;
12853     intptr_t mask = $mask$$constant;
12854     int width = exact_log2(mask+1);
12855     __ ubfx(as_Register($dst$$reg),
12856             as_Register($src$$reg), rshift, width);
12857   %}
12858   ins_pipe(ialu_reg_shift);
12859 %}
12860 
12861 
12862 // This pattern is automatically generated from aarch64_ad.m4
12863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12864 
12865 // We can use ubfiz when masking by a positive number and then left shifting the result.
12866 // We know that the mask is positive because immI_bitmask guarantees it.
12867 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12868 %{
12869   match(Set dst (LShiftI (AndI src mask) lshift));
12870   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12871 
12872   ins_cost(INSN_COST);
12873   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12874   ins_encode %{
12875     int lshift = $lshift$$constant & 31;
12876     intptr_t mask = $mask$$constant;
12877     int width = exact_log2(mask+1);
12878     __ ubfizw(as_Register($dst$$reg),
12879           as_Register($src$$reg), lshift, width);
12880   %}
12881   ins_pipe(ialu_reg_shift);
12882 %}
12883 
12884 // This pattern is automatically generated from aarch64_ad.m4
12885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12886 
12887 // We can use ubfiz when masking by a positive number and then left shifting the result.
12888 // We know that the mask is positive because immL_bitmask guarantees it.
12889 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12890 %{
12891   match(Set dst (LShiftL (AndL src mask) lshift));
12892   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12893 
12894   ins_cost(INSN_COST);
12895   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12896   ins_encode %{
12897     int lshift = $lshift$$constant & 63;
12898     intptr_t mask = $mask$$constant;
12899     int width = exact_log2_long(mask+1);
12900     __ ubfiz(as_Register($dst$$reg),
12901           as_Register($src$$reg), lshift, width);
12902   %}
12903   ins_pipe(ialu_reg_shift);
12904 %}
12905 
12906 // This pattern is automatically generated from aarch64_ad.m4
12907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12908 
12909 // We can use ubfiz when masking by a positive number and then left shifting the result.
12910 // We know that the mask is positive because immI_bitmask guarantees it.
12911 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12912 %{
12913   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12914   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12915 
12916   ins_cost(INSN_COST);
12917   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12918   ins_encode %{
12919     int lshift = $lshift$$constant & 31;
12920     intptr_t mask = $mask$$constant;
12921     int width = exact_log2(mask+1);
12922     __ ubfizw(as_Register($dst$$reg),
12923           as_Register($src$$reg), lshift, width);
12924   %}
12925   ins_pipe(ialu_reg_shift);
12926 %}
12927 
12928 // This pattern is automatically generated from aarch64_ad.m4
12929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12930 
12931 // We can use ubfiz when masking by a positive number and then left shifting the result.
12932 // We know that the mask is positive because immL_bitmask guarantees it.
12933 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12934 %{
12935   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12936   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12937 
12938   ins_cost(INSN_COST);
12939   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12940   ins_encode %{
12941     int lshift = $lshift$$constant & 63;
12942     intptr_t mask = $mask$$constant;
12943     int width = exact_log2_long(mask+1);
12944     __ ubfiz(as_Register($dst$$reg),
12945           as_Register($src$$reg), lshift, width);
12946   %}
12947   ins_pipe(ialu_reg_shift);
12948 %}
12949 
12950 
12951 // This pattern is automatically generated from aarch64_ad.m4
12952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12953 
12954 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12955 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12956 %{
12957   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12958   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12959 
12960   ins_cost(INSN_COST);
12961   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12962   ins_encode %{
12963     int lshift = $lshift$$constant & 63;
12964     intptr_t mask = $mask$$constant;
12965     int width = exact_log2(mask+1);
12966     __ ubfiz(as_Register($dst$$reg),
12967              as_Register($src$$reg), lshift, width);
12968   %}
12969   ins_pipe(ialu_reg_shift);
12970 %}
12971 
12972 // This pattern is automatically generated from aarch64_ad.m4
12973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12974 
12975 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12976 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12977 %{
12978   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12979   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12980 
12981   ins_cost(INSN_COST);
12982   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12983   ins_encode %{
12984     int lshift = $lshift$$constant & 31;
12985     intptr_t mask = $mask$$constant;
12986     int width = exact_log2(mask+1);
12987     __ ubfiz(as_Register($dst$$reg),
12988              as_Register($src$$reg), lshift, width);
12989   %}
12990   ins_pipe(ialu_reg_shift);
12991 %}
12992 
12993 // This pattern is automatically generated from aarch64_ad.m4
12994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12995 
12996 // Can skip int2long conversions after AND with small bitmask
12997 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12998 %{
12999   match(Set dst (ConvI2L (AndI src msk)));
13000   ins_cost(INSN_COST);
13001   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13002   ins_encode %{
13003     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13004   %}
13005   ins_pipe(ialu_reg_shift);
13006 %}
13007 
13008 
13009 // Rotations
13010 // This pattern is automatically generated from aarch64_ad.m4
13011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13012 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13013 %{
13014   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13015   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13016 
13017   ins_cost(INSN_COST);
13018   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13019 
13020   ins_encode %{
13021     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13022             $rshift$$constant & 63);
13023   %}
13024   ins_pipe(ialu_reg_reg_extr);
13025 %}
13026 
13027 
13028 // This pattern is automatically generated from aarch64_ad.m4
13029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13030 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13031 %{
13032   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13033   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13034 
13035   ins_cost(INSN_COST);
13036   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13037 
13038   ins_encode %{
13039     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13040             $rshift$$constant & 31);
13041   %}
13042   ins_pipe(ialu_reg_reg_extr);
13043 %}
13044 
13045 
13046 // This pattern is automatically generated from aarch64_ad.m4
13047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13048 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13049 %{
13050   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13051   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13052 
13053   ins_cost(INSN_COST);
13054   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13055 
13056   ins_encode %{
13057     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13058             $rshift$$constant & 63);
13059   %}
13060   ins_pipe(ialu_reg_reg_extr);
13061 %}
13062 
13063 
13064 // This pattern is automatically generated from aarch64_ad.m4
13065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13066 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13067 %{
13068   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13069   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13070 
13071   ins_cost(INSN_COST);
13072   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13073 
13074   ins_encode %{
13075     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13076             $rshift$$constant & 31);
13077   %}
13078   ins_pipe(ialu_reg_reg_extr);
13079 %}
13080 
13081 
13082 // This pattern is automatically generated from aarch64_ad.m4
13083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13084 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13085 %{
13086   match(Set dst (RotateRight src shift));
13087 
13088   ins_cost(INSN_COST);
13089   format %{ "ror    $dst, $src, $shift" %}
13090 
13091   ins_encode %{
13092      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13093                $shift$$constant & 0x1f);
13094   %}
13095   ins_pipe(ialu_reg_reg_vshift);
13096 %}
13097 
13098 // This pattern is automatically generated from aarch64_ad.m4
13099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13100 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13101 %{
13102   match(Set dst (RotateRight src shift));
13103 
13104   ins_cost(INSN_COST);
13105   format %{ "ror    $dst, $src, $shift" %}
13106 
13107   ins_encode %{
13108      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13109                $shift$$constant & 0x3f);
13110   %}
13111   ins_pipe(ialu_reg_reg_vshift);
13112 %}
13113 
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13117 %{
13118   match(Set dst (RotateRight src shift));
13119 
13120   ins_cost(INSN_COST);
13121   format %{ "ror    $dst, $src, $shift" %}
13122 
13123   ins_encode %{
13124      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13125   %}
13126   ins_pipe(ialu_reg_reg_vshift);
13127 %}
13128 
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13132 %{
13133   match(Set dst (RotateRight src shift));
13134 
13135   ins_cost(INSN_COST);
13136   format %{ "ror    $dst, $src, $shift" %}
13137 
13138   ins_encode %{
13139      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13140   %}
13141   ins_pipe(ialu_reg_reg_vshift);
13142 %}
13143 
13144 // This pattern is automatically generated from aarch64_ad.m4
13145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13146 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13147 %{
13148   match(Set dst (RotateLeft src shift));
13149 
13150   ins_cost(INSN_COST);
13151   format %{ "rol    $dst, $src, $shift" %}
13152 
13153   ins_encode %{
13154      __ subw(rscratch1, zr, as_Register($shift$$reg));
13155      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13156   %}
13157   ins_pipe(ialu_reg_reg_vshift);
13158 %}
13159 
13160 // This pattern is automatically generated from aarch64_ad.m4
13161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13162 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13163 %{
13164   match(Set dst (RotateLeft src shift));
13165 
13166   ins_cost(INSN_COST);
13167   format %{ "rol    $dst, $src, $shift" %}
13168 
13169   ins_encode %{
13170      __ subw(rscratch1, zr, as_Register($shift$$reg));
13171      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13172   %}
13173   ins_pipe(ialu_reg_reg_vshift);
13174 %}
13175 
13176 
13177 // Add/subtract (extended)
13178 
13179 // This pattern is automatically generated from aarch64_ad.m4
13180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13181 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13182 %{
13183   match(Set dst (AddL src1 (ConvI2L src2)));
13184   ins_cost(INSN_COST);
13185   format %{ "add  $dst, $src1, $src2, sxtw" %}
13186 
13187    ins_encode %{
13188      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13189             as_Register($src2$$reg), ext::sxtw);
13190    %}
13191   ins_pipe(ialu_reg_reg);
13192 %}
13193 
13194 // This pattern is automatically generated from aarch64_ad.m4
13195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13196 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13197 %{
13198   match(Set dst (SubL src1 (ConvI2L src2)));
13199   ins_cost(INSN_COST);
13200   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13201 
13202    ins_encode %{
13203      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13204             as_Register($src2$$reg), ext::sxtw);
13205    %}
13206   ins_pipe(ialu_reg_reg);
13207 %}
13208 
13209 // This pattern is automatically generated from aarch64_ad.m4
13210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13211 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13212 %{
13213   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13214   ins_cost(INSN_COST);
13215   format %{ "add  $dst, $src1, $src2, sxth" %}
13216 
13217    ins_encode %{
13218      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13219             as_Register($src2$$reg), ext::sxth);
13220    %}
13221   ins_pipe(ialu_reg_reg);
13222 %}
13223 
13224 // This pattern is automatically generated from aarch64_ad.m4
13225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13226 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13227 %{
13228   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13229   ins_cost(INSN_COST);
13230   format %{ "add  $dst, $src1, $src2, sxtb" %}
13231 
13232    ins_encode %{
13233      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13234             as_Register($src2$$reg), ext::sxtb);
13235    %}
13236   ins_pipe(ialu_reg_reg);
13237 %}
13238 
13239 // This pattern is automatically generated from aarch64_ad.m4
13240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13241 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13242 %{
13243   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13244   ins_cost(INSN_COST);
13245   format %{ "add  $dst, $src1, $src2, uxtb" %}
13246 
13247    ins_encode %{
13248      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13249             as_Register($src2$$reg), ext::uxtb);
13250    %}
13251   ins_pipe(ialu_reg_reg);
13252 %}
13253 
13254 // This pattern is automatically generated from aarch64_ad.m4
13255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13256 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13257 %{
13258   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13259   ins_cost(INSN_COST);
13260   format %{ "add  $dst, $src1, $src2, sxth" %}
13261 
13262    ins_encode %{
13263      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13264             as_Register($src2$$reg), ext::sxth);
13265    %}
13266   ins_pipe(ialu_reg_reg);
13267 %}
13268 
13269 // This pattern is automatically generated from aarch64_ad.m4
13270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13271 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13272 %{
13273   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13274   ins_cost(INSN_COST);
13275   format %{ "add  $dst, $src1, $src2, sxtw" %}
13276 
13277    ins_encode %{
13278      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13279             as_Register($src2$$reg), ext::sxtw);
13280    %}
13281   ins_pipe(ialu_reg_reg);
13282 %}
13283 
13284 // This pattern is automatically generated from aarch64_ad.m4
13285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13286 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13287 %{
13288   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13289   ins_cost(INSN_COST);
13290   format %{ "add  $dst, $src1, $src2, sxtb" %}
13291 
13292    ins_encode %{
13293      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13294             as_Register($src2$$reg), ext::sxtb);
13295    %}
13296   ins_pipe(ialu_reg_reg);
13297 %}
13298 
13299 // This pattern is automatically generated from aarch64_ad.m4
13300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13301 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13302 %{
13303   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13304   ins_cost(INSN_COST);
13305   format %{ "add  $dst, $src1, $src2, uxtb" %}
13306 
13307    ins_encode %{
13308      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13309             as_Register($src2$$reg), ext::uxtb);
13310    %}
13311   ins_pipe(ialu_reg_reg);
13312 %}
13313 
13314 // This pattern is automatically generated from aarch64_ad.m4
13315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13316 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13317 %{
13318   match(Set dst (AddI src1 (AndI src2 mask)));
13319   ins_cost(INSN_COST);
13320   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13321 
13322    ins_encode %{
13323      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13324             as_Register($src2$$reg), ext::uxtb);
13325    %}
13326   ins_pipe(ialu_reg_reg);
13327 %}
13328 
13329 // This pattern is automatically generated from aarch64_ad.m4
13330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13331 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13332 %{
13333   match(Set dst (AddI src1 (AndI src2 mask)));
13334   ins_cost(INSN_COST);
13335   format %{ "addw  $dst, $src1, $src2, uxth" %}
13336 
13337    ins_encode %{
13338      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13339             as_Register($src2$$reg), ext::uxth);
13340    %}
13341   ins_pipe(ialu_reg_reg);
13342 %}
13343 
13344 // This pattern is automatically generated from aarch64_ad.m4
13345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13346 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13347 %{
13348   match(Set dst (AddL src1 (AndL src2 mask)));
13349   ins_cost(INSN_COST);
13350   format %{ "add  $dst, $src1, $src2, uxtb" %}
13351 
13352    ins_encode %{
13353      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13354             as_Register($src2$$reg), ext::uxtb);
13355    %}
13356   ins_pipe(ialu_reg_reg);
13357 %}
13358 
13359 // This pattern is automatically generated from aarch64_ad.m4
13360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13361 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13362 %{
13363   match(Set dst (AddL src1 (AndL src2 mask)));
13364   ins_cost(INSN_COST);
13365   format %{ "add  $dst, $src1, $src2, uxth" %}
13366 
13367    ins_encode %{
13368      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13369             as_Register($src2$$reg), ext::uxth);
13370    %}
13371   ins_pipe(ialu_reg_reg);
13372 %}
13373 
13374 // This pattern is automatically generated from aarch64_ad.m4
13375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13376 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13377 %{
13378   match(Set dst (AddL src1 (AndL src2 mask)));
13379   ins_cost(INSN_COST);
13380   format %{ "add  $dst, $src1, $src2, uxtw" %}
13381 
13382    ins_encode %{
13383      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13384             as_Register($src2$$reg), ext::uxtw);
13385    %}
13386   ins_pipe(ialu_reg_reg);
13387 %}
13388 
13389 // This pattern is automatically generated from aarch64_ad.m4
13390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13391 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13392 %{
13393   match(Set dst (SubI src1 (AndI src2 mask)));
13394   ins_cost(INSN_COST);
13395   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13396 
13397    ins_encode %{
13398      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13399             as_Register($src2$$reg), ext::uxtb);
13400    %}
13401   ins_pipe(ialu_reg_reg);
13402 %}
13403 
13404 // This pattern is automatically generated from aarch64_ad.m4
13405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13406 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13407 %{
13408   match(Set dst (SubI src1 (AndI src2 mask)));
13409   ins_cost(INSN_COST);
13410   format %{ "subw  $dst, $src1, $src2, uxth" %}
13411 
13412    ins_encode %{
13413      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13414             as_Register($src2$$reg), ext::uxth);
13415    %}
13416   ins_pipe(ialu_reg_reg);
13417 %}
13418 
13419 // This pattern is automatically generated from aarch64_ad.m4
13420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13421 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13422 %{
13423   match(Set dst (SubL src1 (AndL src2 mask)));
13424   ins_cost(INSN_COST);
13425   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13426 
13427    ins_encode %{
13428      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13429             as_Register($src2$$reg), ext::uxtb);
13430    %}
13431   ins_pipe(ialu_reg_reg);
13432 %}
13433 
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13437 %{
13438   match(Set dst (SubL src1 (AndL src2 mask)));
13439   ins_cost(INSN_COST);
13440   format %{ "sub  $dst, $src1, $src2, uxth" %}
13441 
13442    ins_encode %{
13443      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13444             as_Register($src2$$reg), ext::uxth);
13445    %}
13446   ins_pipe(ialu_reg_reg);
13447 %}
13448 
13449 // This pattern is automatically generated from aarch64_ad.m4
13450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13451 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13452 %{
13453   match(Set dst (SubL src1 (AndL src2 mask)));
13454   ins_cost(INSN_COST);
13455   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13456 
13457    ins_encode %{
13458      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13459             as_Register($src2$$reg), ext::uxtw);
13460    %}
13461   ins_pipe(ialu_reg_reg);
13462 %}
13463 
13464 
13465 // This pattern is automatically generated from aarch64_ad.m4
13466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13467 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13468 %{
13469   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13470   ins_cost(1.9 * INSN_COST);
13471   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13472 
13473    ins_encode %{
13474      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13475             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13476    %}
13477   ins_pipe(ialu_reg_reg_shift);
13478 %}
13479 
13480 // This pattern is automatically generated from aarch64_ad.m4
13481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13482 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13483 %{
13484   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13485   ins_cost(1.9 * INSN_COST);
13486   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13487 
13488    ins_encode %{
13489      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13490             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13491    %}
13492   ins_pipe(ialu_reg_reg_shift);
13493 %}
13494 
13495 // This pattern is automatically generated from aarch64_ad.m4
13496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13497 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13498 %{
13499   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13500   ins_cost(1.9 * INSN_COST);
13501   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13502 
13503    ins_encode %{
13504      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13505             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13506    %}
13507   ins_pipe(ialu_reg_reg_shift);
13508 %}
13509 
13510 // This pattern is automatically generated from aarch64_ad.m4
13511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13512 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13513 %{
13514   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13515   ins_cost(1.9 * INSN_COST);
13516   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13517 
13518    ins_encode %{
13519      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13520             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13521    %}
13522   ins_pipe(ialu_reg_reg_shift);
13523 %}
13524 
13525 // This pattern is automatically generated from aarch64_ad.m4
13526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13527 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13528 %{
13529   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13530   ins_cost(1.9 * INSN_COST);
13531   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13532 
13533    ins_encode %{
13534      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13535             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13536    %}
13537   ins_pipe(ialu_reg_reg_shift);
13538 %}
13539 
13540 // This pattern is automatically generated from aarch64_ad.m4
13541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13542 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13543 %{
13544   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13545   ins_cost(1.9 * INSN_COST);
13546   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13547 
13548    ins_encode %{
13549      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13550             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13551    %}
13552   ins_pipe(ialu_reg_reg_shift);
13553 %}
13554 
13555 // This pattern is automatically generated from aarch64_ad.m4
13556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13557 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13558 %{
13559   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13560   ins_cost(1.9 * INSN_COST);
13561   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13562 
13563    ins_encode %{
13564      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13565             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13566    %}
13567   ins_pipe(ialu_reg_reg_shift);
13568 %}
13569 
13570 // This pattern is automatically generated from aarch64_ad.m4
13571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13572 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13573 %{
13574   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13575   ins_cost(1.9 * INSN_COST);
13576   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13577 
13578    ins_encode %{
13579      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13580             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13581    %}
13582   ins_pipe(ialu_reg_reg_shift);
13583 %}
13584 
13585 // This pattern is automatically generated from aarch64_ad.m4
13586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13587 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13588 %{
13589   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13590   ins_cost(1.9 * INSN_COST);
13591   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13592 
13593    ins_encode %{
13594      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13595             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13596    %}
13597   ins_pipe(ialu_reg_reg_shift);
13598 %}
13599 
13600 // This pattern is automatically generated from aarch64_ad.m4
13601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13602 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13603 %{
13604   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13605   ins_cost(1.9 * INSN_COST);
13606   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13607 
13608    ins_encode %{
13609      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13610             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13611    %}
13612   ins_pipe(ialu_reg_reg_shift);
13613 %}
13614 
13615 // This pattern is automatically generated from aarch64_ad.m4
13616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13617 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13618 %{
13619   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13620   ins_cost(1.9 * INSN_COST);
13621   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13622 
13623    ins_encode %{
13624      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13625             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13626    %}
13627   ins_pipe(ialu_reg_reg_shift);
13628 %}
13629 
13630 // This pattern is automatically generated from aarch64_ad.m4
13631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13632 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13633 %{
13634   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13635   ins_cost(1.9 * INSN_COST);
13636   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13637 
13638    ins_encode %{
13639      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13640             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13641    %}
13642   ins_pipe(ialu_reg_reg_shift);
13643 %}
13644 
13645 // This pattern is automatically generated from aarch64_ad.m4
13646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13647 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13648 %{
13649   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13650   ins_cost(1.9 * INSN_COST);
13651   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13652 
13653    ins_encode %{
13654      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13655             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13656    %}
13657   ins_pipe(ialu_reg_reg_shift);
13658 %}
13659 
13660 // This pattern is automatically generated from aarch64_ad.m4
13661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13662 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13663 %{
13664   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13665   ins_cost(1.9 * INSN_COST);
13666   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13667 
13668    ins_encode %{
13669      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13670             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13671    %}
13672   ins_pipe(ialu_reg_reg_shift);
13673 %}
13674 
13675 // This pattern is automatically generated from aarch64_ad.m4
13676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13677 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13678 %{
13679   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13680   ins_cost(1.9 * INSN_COST);
13681   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13682 
13683    ins_encode %{
13684      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13685             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13686    %}
13687   ins_pipe(ialu_reg_reg_shift);
13688 %}
13689 
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13693 %{
13694   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13695   ins_cost(1.9 * INSN_COST);
13696   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13697 
13698    ins_encode %{
13699      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13700             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13701    %}
13702   ins_pipe(ialu_reg_reg_shift);
13703 %}
13704 
13705 // This pattern is automatically generated from aarch64_ad.m4
13706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13707 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13708 %{
13709   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13710   ins_cost(1.9 * INSN_COST);
13711   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13712 
13713    ins_encode %{
13714      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13715             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13716    %}
13717   ins_pipe(ialu_reg_reg_shift);
13718 %}
13719 
13720 // This pattern is automatically generated from aarch64_ad.m4
13721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13722 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13723 %{
13724   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13725   ins_cost(1.9 * INSN_COST);
13726   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13727 
13728    ins_encode %{
13729      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13730             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13731    %}
13732   ins_pipe(ialu_reg_reg_shift);
13733 %}
13734 
13735 // This pattern is automatically generated from aarch64_ad.m4
13736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13737 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13738 %{
13739   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13740   ins_cost(1.9 * INSN_COST);
13741   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13742 
13743    ins_encode %{
13744      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13745             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13746    %}
13747   ins_pipe(ialu_reg_reg_shift);
13748 %}
13749 
13750 // This pattern is automatically generated from aarch64_ad.m4
13751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13752 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13753 %{
13754   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13755   ins_cost(1.9 * INSN_COST);
13756   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13757 
13758    ins_encode %{
13759      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13760             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13761    %}
13762   ins_pipe(ialu_reg_reg_shift);
13763 %}
13764 
13765 // This pattern is automatically generated from aarch64_ad.m4
13766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13767 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13768 %{
13769   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13770   ins_cost(1.9 * INSN_COST);
13771   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13772 
13773    ins_encode %{
13774      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13775             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13776    %}
13777   ins_pipe(ialu_reg_reg_shift);
13778 %}
13779 
13780 // This pattern is automatically generated from aarch64_ad.m4
13781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13782 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13783 %{
13784   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13785   ins_cost(1.9 * INSN_COST);
13786   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13787 
13788    ins_encode %{
13789      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13790             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13791    %}
13792   ins_pipe(ialu_reg_reg_shift);
13793 %}
13794 
13795 
13796 
13797 // END This section of the file is automatically generated. Do not edit --------------
13798 
13799 
13800 // ============================================================================
13801 // Floating Point Arithmetic Instructions
13802 
13803 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13804   match(Set dst (AddF src1 src2));
13805 
13806   ins_cost(INSN_COST * 5);
13807   format %{ "fadds   $dst, $src1, $src2" %}
13808 
13809   ins_encode %{
13810     __ fadds(as_FloatRegister($dst$$reg),
13811              as_FloatRegister($src1$$reg),
13812              as_FloatRegister($src2$$reg));
13813   %}
13814 
13815   ins_pipe(fp_dop_reg_reg_s);
13816 %}
13817 
13818 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13819   match(Set dst (AddD src1 src2));
13820 
13821   ins_cost(INSN_COST * 5);
13822   format %{ "faddd   $dst, $src1, $src2" %}
13823 
13824   ins_encode %{
13825     __ faddd(as_FloatRegister($dst$$reg),
13826              as_FloatRegister($src1$$reg),
13827              as_FloatRegister($src2$$reg));
13828   %}
13829 
13830   ins_pipe(fp_dop_reg_reg_d);
13831 %}
13832 
13833 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13834   match(Set dst (SubF src1 src2));
13835 
13836   ins_cost(INSN_COST * 5);
13837   format %{ "fsubs   $dst, $src1, $src2" %}
13838 
13839   ins_encode %{
13840     __ fsubs(as_FloatRegister($dst$$reg),
13841              as_FloatRegister($src1$$reg),
13842              as_FloatRegister($src2$$reg));
13843   %}
13844 
13845   ins_pipe(fp_dop_reg_reg_s);
13846 %}
13847 
13848 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13849   match(Set dst (SubD src1 src2));
13850 
13851   ins_cost(INSN_COST * 5);
13852   format %{ "fsubd   $dst, $src1, $src2" %}
13853 
13854   ins_encode %{
13855     __ fsubd(as_FloatRegister($dst$$reg),
13856              as_FloatRegister($src1$$reg),
13857              as_FloatRegister($src2$$reg));
13858   %}
13859 
13860   ins_pipe(fp_dop_reg_reg_d);
13861 %}
13862 
13863 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13864   match(Set dst (MulF src1 src2));
13865 
13866   ins_cost(INSN_COST * 6);
13867   format %{ "fmuls   $dst, $src1, $src2" %}
13868 
13869   ins_encode %{
13870     __ fmuls(as_FloatRegister($dst$$reg),
13871              as_FloatRegister($src1$$reg),
13872              as_FloatRegister($src2$$reg));
13873   %}
13874 
13875   ins_pipe(fp_dop_reg_reg_s);
13876 %}
13877 
13878 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13879   match(Set dst (MulD src1 src2));
13880 
13881   ins_cost(INSN_COST * 6);
13882   format %{ "fmuld   $dst, $src1, $src2" %}
13883 
13884   ins_encode %{
13885     __ fmuld(as_FloatRegister($dst$$reg),
13886              as_FloatRegister($src1$$reg),
13887              as_FloatRegister($src2$$reg));
13888   %}
13889 
13890   ins_pipe(fp_dop_reg_reg_d);
13891 %}
13892 
13893 // src1 * src2 + src3
13894 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13895   predicate(UseFMA);
13896   match(Set dst (FmaF src3 (Binary src1 src2)));
13897 
13898   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13899 
13900   ins_encode %{
13901     __ fmadds(as_FloatRegister($dst$$reg),
13902              as_FloatRegister($src1$$reg),
13903              as_FloatRegister($src2$$reg),
13904              as_FloatRegister($src3$$reg));
13905   %}
13906 
13907   ins_pipe(pipe_class_default);
13908 %}
13909 
13910 // src1 * src2 + src3
13911 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13912   predicate(UseFMA);
13913   match(Set dst (FmaD src3 (Binary src1 src2)));
13914 
13915   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13916 
13917   ins_encode %{
13918     __ fmaddd(as_FloatRegister($dst$$reg),
13919              as_FloatRegister($src1$$reg),
13920              as_FloatRegister($src2$$reg),
13921              as_FloatRegister($src3$$reg));
13922   %}
13923 
13924   ins_pipe(pipe_class_default);
13925 %}
13926 
13927 // -src1 * src2 + src3
13928 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13929   predicate(UseFMA);
13930   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13931   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13932 
13933   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13934 
13935   ins_encode %{
13936     __ fmsubs(as_FloatRegister($dst$$reg),
13937               as_FloatRegister($src1$$reg),
13938               as_FloatRegister($src2$$reg),
13939               as_FloatRegister($src3$$reg));
13940   %}
13941 
13942   ins_pipe(pipe_class_default);
13943 %}
13944 
13945 // -src1 * src2 + src3
13946 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13947   predicate(UseFMA);
13948   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13949   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13950 
13951   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13952 
13953   ins_encode %{
13954     __ fmsubd(as_FloatRegister($dst$$reg),
13955               as_FloatRegister($src1$$reg),
13956               as_FloatRegister($src2$$reg),
13957               as_FloatRegister($src3$$reg));
13958   %}
13959 
13960   ins_pipe(pipe_class_default);
13961 %}
13962 
13963 // -src1 * src2 - src3
13964 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13965   predicate(UseFMA);
13966   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13967   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13968 
13969   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13970 
13971   ins_encode %{
13972     __ fnmadds(as_FloatRegister($dst$$reg),
13973                as_FloatRegister($src1$$reg),
13974                as_FloatRegister($src2$$reg),
13975                as_FloatRegister($src3$$reg));
13976   %}
13977 
13978   ins_pipe(pipe_class_default);
13979 %}
13980 
13981 // -src1 * src2 - src3
13982 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13983   predicate(UseFMA);
13984   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13985   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13986 
13987   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13988 
13989   ins_encode %{
13990     __ fnmaddd(as_FloatRegister($dst$$reg),
13991                as_FloatRegister($src1$$reg),
13992                as_FloatRegister($src2$$reg),
13993                as_FloatRegister($src3$$reg));
13994   %}
13995 
13996   ins_pipe(pipe_class_default);
13997 %}
13998 
13999 // src1 * src2 - src3
14000 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14001   predicate(UseFMA);
14002   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14003 
14004   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14005 
14006   ins_encode %{
14007     __ fnmsubs(as_FloatRegister($dst$$reg),
14008                as_FloatRegister($src1$$reg),
14009                as_FloatRegister($src2$$reg),
14010                as_FloatRegister($src3$$reg));
14011   %}
14012 
14013   ins_pipe(pipe_class_default);
14014 %}
14015 
14016 // src1 * src2 - src3
14017 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14018   predicate(UseFMA);
14019   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14020 
14021   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14022 
14023   ins_encode %{
14024   // n.b. insn name should be fnmsubd
14025     __ fnmsub(as_FloatRegister($dst$$reg),
14026               as_FloatRegister($src1$$reg),
14027               as_FloatRegister($src2$$reg),
14028               as_FloatRegister($src3$$reg));
14029   %}
14030 
14031   ins_pipe(pipe_class_default);
14032 %}
14033 
14034 
14035 // Math.max(FF)F
14036 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14037   match(Set dst (MaxF src1 src2));
14038 
14039   format %{ "fmaxs   $dst, $src1, $src2" %}
14040   ins_encode %{
14041     __ fmaxs(as_FloatRegister($dst$$reg),
14042              as_FloatRegister($src1$$reg),
14043              as_FloatRegister($src2$$reg));
14044   %}
14045 
14046   ins_pipe(fp_dop_reg_reg_s);
14047 %}
14048 
14049 // Math.min(FF)F
14050 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14051   match(Set dst (MinF src1 src2));
14052 
14053   format %{ "fmins   $dst, $src1, $src2" %}
14054   ins_encode %{
14055     __ fmins(as_FloatRegister($dst$$reg),
14056              as_FloatRegister($src1$$reg),
14057              as_FloatRegister($src2$$reg));
14058   %}
14059 
14060   ins_pipe(fp_dop_reg_reg_s);
14061 %}
14062 
14063 // Math.max(DD)D
14064 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14065   match(Set dst (MaxD src1 src2));
14066 
14067   format %{ "fmaxd   $dst, $src1, $src2" %}
14068   ins_encode %{
14069     __ fmaxd(as_FloatRegister($dst$$reg),
14070              as_FloatRegister($src1$$reg),
14071              as_FloatRegister($src2$$reg));
14072   %}
14073 
14074   ins_pipe(fp_dop_reg_reg_d);
14075 %}
14076 
14077 // Math.min(DD)D
14078 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14079   match(Set dst (MinD src1 src2));
14080 
14081   format %{ "fmind   $dst, $src1, $src2" %}
14082   ins_encode %{
14083     __ fmind(as_FloatRegister($dst$$reg),
14084              as_FloatRegister($src1$$reg),
14085              as_FloatRegister($src2$$reg));
14086   %}
14087 
14088   ins_pipe(fp_dop_reg_reg_d);
14089 %}
14090 
14091 
14092 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14093   match(Set dst (DivF src1  src2));
14094 
14095   ins_cost(INSN_COST * 18);
14096   format %{ "fdivs   $dst, $src1, $src2" %}
14097 
14098   ins_encode %{
14099     __ fdivs(as_FloatRegister($dst$$reg),
14100              as_FloatRegister($src1$$reg),
14101              as_FloatRegister($src2$$reg));
14102   %}
14103 
14104   ins_pipe(fp_div_s);
14105 %}
14106 
14107 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14108   match(Set dst (DivD src1  src2));
14109 
14110   ins_cost(INSN_COST * 32);
14111   format %{ "fdivd   $dst, $src1, $src2" %}
14112 
14113   ins_encode %{
14114     __ fdivd(as_FloatRegister($dst$$reg),
14115              as_FloatRegister($src1$$reg),
14116              as_FloatRegister($src2$$reg));
14117   %}
14118 
14119   ins_pipe(fp_div_d);
14120 %}
14121 
14122 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14123   match(Set dst (NegF src));
14124 
14125   ins_cost(INSN_COST * 3);
14126   format %{ "fneg   $dst, $src" %}
14127 
14128   ins_encode %{
14129     __ fnegs(as_FloatRegister($dst$$reg),
14130              as_FloatRegister($src$$reg));
14131   %}
14132 
14133   ins_pipe(fp_uop_s);
14134 %}
14135 
14136 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14137   match(Set dst (NegD src));
14138 
14139   ins_cost(INSN_COST * 3);
14140   format %{ "fnegd   $dst, $src" %}
14141 
14142   ins_encode %{
14143     __ fnegd(as_FloatRegister($dst$$reg),
14144              as_FloatRegister($src$$reg));
14145   %}
14146 
14147   ins_pipe(fp_uop_d);
14148 %}
14149 
14150 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14151 %{
14152   match(Set dst (AbsI src));
14153 
14154   effect(KILL cr);
14155   ins_cost(INSN_COST * 2);
14156   format %{ "cmpw  $src, zr\n\t"
14157             "cnegw $dst, $src, Assembler::LT\t# int abs"
14158   %}
14159 
14160   ins_encode %{
14161     __ cmpw(as_Register($src$$reg), zr);
14162     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14163   %}
14164   ins_pipe(pipe_class_default);
14165 %}
14166 
14167 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14168 %{
14169   match(Set dst (AbsL src));
14170 
14171   effect(KILL cr);
14172   ins_cost(INSN_COST * 2);
14173   format %{ "cmp  $src, zr\n\t"
14174             "cneg $dst, $src, Assembler::LT\t# long abs"
14175   %}
14176 
14177   ins_encode %{
14178     __ cmp(as_Register($src$$reg), zr);
14179     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14180   %}
14181   ins_pipe(pipe_class_default);
14182 %}
14183 
14184 instruct absF_reg(vRegF dst, vRegF src) %{
14185   match(Set dst (AbsF src));
14186 
14187   ins_cost(INSN_COST * 3);
14188   format %{ "fabss   $dst, $src" %}
14189   ins_encode %{
14190     __ fabss(as_FloatRegister($dst$$reg),
14191              as_FloatRegister($src$$reg));
14192   %}
14193 
14194   ins_pipe(fp_uop_s);
14195 %}
14196 
14197 instruct absD_reg(vRegD dst, vRegD src) %{
14198   match(Set dst (AbsD src));
14199 
14200   ins_cost(INSN_COST * 3);
14201   format %{ "fabsd   $dst, $src" %}
14202   ins_encode %{
14203     __ fabsd(as_FloatRegister($dst$$reg),
14204              as_FloatRegister($src$$reg));
14205   %}
14206 
14207   ins_pipe(fp_uop_d);
14208 %}
14209 
14210 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14211   match(Set dst (AbsF (SubF src1 src2)));
14212 
14213   ins_cost(INSN_COST * 3);
14214   format %{ "fabds   $dst, $src1, $src2" %}
14215   ins_encode %{
14216     __ fabds(as_FloatRegister($dst$$reg),
14217              as_FloatRegister($src1$$reg),
14218              as_FloatRegister($src2$$reg));
14219   %}
14220 
14221   ins_pipe(fp_uop_s);
14222 %}
14223 
14224 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14225   match(Set dst (AbsD (SubD src1 src2)));
14226 
14227   ins_cost(INSN_COST * 3);
14228   format %{ "fabdd   $dst, $src1, $src2" %}
14229   ins_encode %{
14230     __ fabdd(as_FloatRegister($dst$$reg),
14231              as_FloatRegister($src1$$reg),
14232              as_FloatRegister($src2$$reg));
14233   %}
14234 
14235   ins_pipe(fp_uop_d);
14236 %}
14237 
14238 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14239   match(Set dst (SqrtD src));
14240 
14241   ins_cost(INSN_COST * 50);
14242   format %{ "fsqrtd  $dst, $src" %}
14243   ins_encode %{
14244     __ fsqrtd(as_FloatRegister($dst$$reg),
14245              as_FloatRegister($src$$reg));
14246   %}
14247 
14248   ins_pipe(fp_div_s);
14249 %}
14250 
14251 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14252   match(Set dst (SqrtF src));
14253 
14254   ins_cost(INSN_COST * 50);
14255   format %{ "fsqrts  $dst, $src" %}
14256   ins_encode %{
14257     __ fsqrts(as_FloatRegister($dst$$reg),
14258              as_FloatRegister($src$$reg));
14259   %}
14260 
14261   ins_pipe(fp_div_d);
14262 %}
14263 
14264 // Math.rint, floor, ceil
14265 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14266   match(Set dst (RoundDoubleMode src rmode));
14267   format %{ "frint  $dst, $src, $rmode" %}
14268   ins_encode %{
14269     switch ($rmode$$constant) {
14270       case RoundDoubleModeNode::rmode_rint:
14271         __ frintnd(as_FloatRegister($dst$$reg),
14272                    as_FloatRegister($src$$reg));
14273         break;
14274       case RoundDoubleModeNode::rmode_floor:
14275         __ frintmd(as_FloatRegister($dst$$reg),
14276                    as_FloatRegister($src$$reg));
14277         break;
14278       case RoundDoubleModeNode::rmode_ceil:
14279         __ frintpd(as_FloatRegister($dst$$reg),
14280                    as_FloatRegister($src$$reg));
14281         break;
14282     }
14283   %}
14284   ins_pipe(fp_uop_d);
14285 %}
14286 
14287 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14288   match(Set dst (CopySignD src1 (Binary src2 zero)));
14289   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14290   format %{ "CopySignD  $dst $src1 $src2" %}
14291   ins_encode %{
14292     FloatRegister dst = as_FloatRegister($dst$$reg),
14293                   src1 = as_FloatRegister($src1$$reg),
14294                   src2 = as_FloatRegister($src2$$reg),
14295                   zero = as_FloatRegister($zero$$reg);
14296     __ fnegd(dst, zero);
14297     __ bsl(dst, __ T8B, src2, src1);
14298   %}
14299   ins_pipe(fp_uop_d);
14300 %}
14301 
14302 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14303   match(Set dst (CopySignF src1 src2));
14304   effect(TEMP_DEF dst, USE src1, USE src2);
14305   format %{ "CopySignF  $dst $src1 $src2" %}
14306   ins_encode %{
14307     FloatRegister dst = as_FloatRegister($dst$$reg),
14308                   src1 = as_FloatRegister($src1$$reg),
14309                   src2 = as_FloatRegister($src2$$reg);
14310     __ movi(dst, __ T2S, 0x80, 24);
14311     __ bsl(dst, __ T8B, src2, src1);
14312   %}
14313   ins_pipe(fp_uop_d);
14314 %}
14315 
14316 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14317   match(Set dst (SignumD src (Binary zero one)));
14318   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14319   format %{ "signumD  $dst, $src" %}
14320   ins_encode %{
14321     FloatRegister src = as_FloatRegister($src$$reg),
14322                   dst = as_FloatRegister($dst$$reg),
14323                   zero = as_FloatRegister($zero$$reg),
14324                   one = as_FloatRegister($one$$reg);
14325     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14326     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14327     // Bit selection instruction gets bit from "one" for each enabled bit in
14328     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14329     // NaN the whole "src" will be copied because "dst" is zero. For all other
14330     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14331     // from "src", and all other bits are copied from 1.0.
14332     __ bsl(dst, __ T8B, one, src);
14333   %}
14334   ins_pipe(fp_uop_d);
14335 %}
14336 
14337 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14338   match(Set dst (SignumF src (Binary zero one)));
14339   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14340   format %{ "signumF  $dst, $src" %}
14341   ins_encode %{
14342     FloatRegister src = as_FloatRegister($src$$reg),
14343                   dst = as_FloatRegister($dst$$reg),
14344                   zero = as_FloatRegister($zero$$reg),
14345                   one = as_FloatRegister($one$$reg);
14346     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14347     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14348     // Bit selection instruction gets bit from "one" for each enabled bit in
14349     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14350     // NaN the whole "src" will be copied because "dst" is zero. For all other
14351     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14352     // from "src", and all other bits are copied from 1.0.
14353     __ bsl(dst, __ T8B, one, src);
14354   %}
14355   ins_pipe(fp_uop_d);
14356 %}
14357 
14358 // ============================================================================
14359 // Logical Instructions
14360 
14361 // Integer Logical Instructions
14362 
14363 // And Instructions
14364 
14365 
14366 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14367   match(Set dst (AndI src1 src2));
14368 
14369   format %{ "andw  $dst, $src1, $src2\t# int" %}
14370 
14371   ins_cost(INSN_COST);
14372   ins_encode %{
14373     __ andw(as_Register($dst$$reg),
14374             as_Register($src1$$reg),
14375             as_Register($src2$$reg));
14376   %}
14377 
14378   ins_pipe(ialu_reg_reg);
14379 %}
14380 
14381 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14382   match(Set dst (AndI src1 src2));
14383 
14384   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14385 
14386   ins_cost(INSN_COST);
14387   ins_encode %{
14388     __ andw(as_Register($dst$$reg),
14389             as_Register($src1$$reg),
14390             (uint64_t)($src2$$constant));
14391   %}
14392 
14393   ins_pipe(ialu_reg_imm);
14394 %}
14395 
14396 // Or Instructions
14397 
14398 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14399   match(Set dst (OrI src1 src2));
14400 
14401   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14402 
14403   ins_cost(INSN_COST);
14404   ins_encode %{
14405     __ orrw(as_Register($dst$$reg),
14406             as_Register($src1$$reg),
14407             as_Register($src2$$reg));
14408   %}
14409 
14410   ins_pipe(ialu_reg_reg);
14411 %}
14412 
14413 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14414   match(Set dst (OrI src1 src2));
14415 
14416   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14417 
14418   ins_cost(INSN_COST);
14419   ins_encode %{
14420     __ orrw(as_Register($dst$$reg),
14421             as_Register($src1$$reg),
14422             (uint64_t)($src2$$constant));
14423   %}
14424 
14425   ins_pipe(ialu_reg_imm);
14426 %}
14427 
14428 // Xor Instructions
14429 
14430 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14431   match(Set dst (XorI src1 src2));
14432 
14433   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14434 
14435   ins_cost(INSN_COST);
14436   ins_encode %{
14437     __ eorw(as_Register($dst$$reg),
14438             as_Register($src1$$reg),
14439             as_Register($src2$$reg));
14440   %}
14441 
14442   ins_pipe(ialu_reg_reg);
14443 %}
14444 
14445 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14446   match(Set dst (XorI src1 src2));
14447 
14448   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14449 
14450   ins_cost(INSN_COST);
14451   ins_encode %{
14452     __ eorw(as_Register($dst$$reg),
14453             as_Register($src1$$reg),
14454             (uint64_t)($src2$$constant));
14455   %}
14456 
14457   ins_pipe(ialu_reg_imm);
14458 %}
14459 
14460 // Long Logical Instructions
14461 // TODO
14462 
14463 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14464   match(Set dst (AndL src1 src2));
14465 
14466   format %{ "and  $dst, $src1, $src2\t# int" %}
14467 
14468   ins_cost(INSN_COST);
14469   ins_encode %{
14470     __ andr(as_Register($dst$$reg),
14471             as_Register($src1$$reg),
14472             as_Register($src2$$reg));
14473   %}
14474 
14475   ins_pipe(ialu_reg_reg);
14476 %}
14477 
14478 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14479   match(Set dst (AndL src1 src2));
14480 
14481   format %{ "and  $dst, $src1, $src2\t# int" %}
14482 
14483   ins_cost(INSN_COST);
14484   ins_encode %{
14485     __ andr(as_Register($dst$$reg),
14486             as_Register($src1$$reg),
14487             (uint64_t)($src2$$constant));
14488   %}
14489 
14490   ins_pipe(ialu_reg_imm);
14491 %}
14492 
14493 // Or Instructions
14494 
14495 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14496   match(Set dst (OrL src1 src2));
14497 
14498   format %{ "orr  $dst, $src1, $src2\t# int" %}
14499 
14500   ins_cost(INSN_COST);
14501   ins_encode %{
14502     __ orr(as_Register($dst$$reg),
14503            as_Register($src1$$reg),
14504            as_Register($src2$$reg));
14505   %}
14506 
14507   ins_pipe(ialu_reg_reg);
14508 %}
14509 
14510 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14511   match(Set dst (OrL src1 src2));
14512 
14513   format %{ "orr  $dst, $src1, $src2\t# int" %}
14514 
14515   ins_cost(INSN_COST);
14516   ins_encode %{
14517     __ orr(as_Register($dst$$reg),
14518            as_Register($src1$$reg),
14519            (uint64_t)($src2$$constant));
14520   %}
14521 
14522   ins_pipe(ialu_reg_imm);
14523 %}
14524 
14525 // Xor Instructions
14526 
14527 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14528   match(Set dst (XorL src1 src2));
14529 
14530   format %{ "eor  $dst, $src1, $src2\t# int" %}
14531 
14532   ins_cost(INSN_COST);
14533   ins_encode %{
14534     __ eor(as_Register($dst$$reg),
14535            as_Register($src1$$reg),
14536            as_Register($src2$$reg));
14537   %}
14538 
14539   ins_pipe(ialu_reg_reg);
14540 %}
14541 
14542 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14543   match(Set dst (XorL src1 src2));
14544 
14545   ins_cost(INSN_COST);
14546   format %{ "eor  $dst, $src1, $src2\t# int" %}
14547 
14548   ins_encode %{
14549     __ eor(as_Register($dst$$reg),
14550            as_Register($src1$$reg),
14551            (uint64_t)($src2$$constant));
14552   %}
14553 
14554   ins_pipe(ialu_reg_imm);
14555 %}
14556 
14557 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14558 %{
14559   match(Set dst (ConvI2L src));
14560 
14561   ins_cost(INSN_COST);
14562   format %{ "sxtw  $dst, $src\t# i2l" %}
14563   ins_encode %{
14564     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14565   %}
14566   ins_pipe(ialu_reg_shift);
14567 %}
14568 
14569 // this pattern occurs in bigmath arithmetic
14570 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14571 %{
14572   match(Set dst (AndL (ConvI2L src) mask));
14573 
14574   ins_cost(INSN_COST);
14575   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14576   ins_encode %{
14577     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14578   %}
14579 
14580   ins_pipe(ialu_reg_shift);
14581 %}
14582 
14583 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14584   match(Set dst (ConvL2I src));
14585 
14586   ins_cost(INSN_COST);
14587   format %{ "movw  $dst, $src \t// l2i" %}
14588 
14589   ins_encode %{
14590     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14591   %}
14592 
14593   ins_pipe(ialu_reg);
14594 %}
14595 
14596 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14597 %{
14598   match(Set dst (Conv2B src));
14599   effect(KILL cr);
14600 
14601   format %{
14602     "cmpw $src, zr\n\t"
14603     "cset $dst, ne"
14604   %}
14605 
14606   ins_encode %{
14607     __ cmpw(as_Register($src$$reg), zr);
14608     __ cset(as_Register($dst$$reg), Assembler::NE);
14609   %}
14610 
14611   ins_pipe(ialu_reg);
14612 %}
14613 
14614 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14615 %{
14616   match(Set dst (Conv2B src));
14617   effect(KILL cr);
14618 
14619   format %{
14620     "cmp  $src, zr\n\t"
14621     "cset $dst, ne"
14622   %}
14623 
14624   ins_encode %{
14625     __ cmp(as_Register($src$$reg), zr);
14626     __ cset(as_Register($dst$$reg), Assembler::NE);
14627   %}
14628 
14629   ins_pipe(ialu_reg);
14630 %}
14631 
14632 instruct convD2F_reg(vRegF dst, vRegD src) %{
14633   match(Set dst (ConvD2F src));
14634 
14635   ins_cost(INSN_COST * 5);
14636   format %{ "fcvtd  $dst, $src \t// d2f" %}
14637 
14638   ins_encode %{
14639     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14640   %}
14641 
14642   ins_pipe(fp_d2f);
14643 %}
14644 
14645 instruct convF2D_reg(vRegD dst, vRegF src) %{
14646   match(Set dst (ConvF2D src));
14647 
14648   ins_cost(INSN_COST * 5);
14649   format %{ "fcvts  $dst, $src \t// f2d" %}
14650 
14651   ins_encode %{
14652     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14653   %}
14654 
14655   ins_pipe(fp_f2d);
14656 %}
14657 
14658 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14659   match(Set dst (ConvF2I src));
14660 
14661   ins_cost(INSN_COST * 5);
14662   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14663 
14664   ins_encode %{
14665     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14666   %}
14667 
14668   ins_pipe(fp_f2i);
14669 %}
14670 
14671 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14672   match(Set dst (ConvF2L src));
14673 
14674   ins_cost(INSN_COST * 5);
14675   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14676 
14677   ins_encode %{
14678     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14679   %}
14680 
14681   ins_pipe(fp_f2l);
14682 %}
14683 
14684 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14685   match(Set dst (ConvI2F src));
14686 
14687   ins_cost(INSN_COST * 5);
14688   format %{ "scvtfws  $dst, $src \t// i2f" %}
14689 
14690   ins_encode %{
14691     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14692   %}
14693 
14694   ins_pipe(fp_i2f);
14695 %}
14696 
14697 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14698   match(Set dst (ConvL2F src));
14699 
14700   ins_cost(INSN_COST * 5);
14701   format %{ "scvtfs  $dst, $src \t// l2f" %}
14702 
14703   ins_encode %{
14704     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14705   %}
14706 
14707   ins_pipe(fp_l2f);
14708 %}
14709 
14710 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14711   match(Set dst (ConvD2I src));
14712 
14713   ins_cost(INSN_COST * 5);
14714   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14715 
14716   ins_encode %{
14717     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14718   %}
14719 
14720   ins_pipe(fp_d2i);
14721 %}
14722 
14723 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14724   match(Set dst (ConvD2L src));
14725 
14726   ins_cost(INSN_COST * 5);
14727   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14728 
14729   ins_encode %{
14730     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14731   %}
14732 
14733   ins_pipe(fp_d2l);
14734 %}
14735 
14736 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14737   match(Set dst (ConvI2D src));
14738 
14739   ins_cost(INSN_COST * 5);
14740   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14741 
14742   ins_encode %{
14743     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14744   %}
14745 
14746   ins_pipe(fp_i2d);
14747 %}
14748 
14749 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14750   match(Set dst (ConvL2D src));
14751 
14752   ins_cost(INSN_COST * 5);
14753   format %{ "scvtfd  $dst, $src \t// l2d" %}
14754 
14755   ins_encode %{
14756     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14757   %}
14758 
14759   ins_pipe(fp_l2d);
14760 %}
14761 
14762 // stack <-> reg and reg <-> reg shuffles with no conversion
14763 
14764 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14765 
14766   match(Set dst (MoveF2I src));
14767 
14768   effect(DEF dst, USE src);
14769 
14770   ins_cost(4 * INSN_COST);
14771 
14772   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14773 
14774   ins_encode %{
14775     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14776   %}
14777 
14778   ins_pipe(iload_reg_reg);
14779 
14780 %}
14781 
14782 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14783 
14784   match(Set dst (MoveI2F src));
14785 
14786   effect(DEF dst, USE src);
14787 
14788   ins_cost(4 * INSN_COST);
14789 
14790   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14791 
14792   ins_encode %{
14793     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14794   %}
14795 
14796   ins_pipe(pipe_class_memory);
14797 
14798 %}
14799 
14800 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14801 
14802   match(Set dst (MoveD2L src));
14803 
14804   effect(DEF dst, USE src);
14805 
14806   ins_cost(4 * INSN_COST);
14807 
14808   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14809 
14810   ins_encode %{
14811     __ ldr($dst$$Register, Address(sp, $src$$disp));
14812   %}
14813 
14814   ins_pipe(iload_reg_reg);
14815 
14816 %}
14817 
14818 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14819 
14820   match(Set dst (MoveL2D src));
14821 
14822   effect(DEF dst, USE src);
14823 
14824   ins_cost(4 * INSN_COST);
14825 
14826   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14827 
14828   ins_encode %{
14829     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14830   %}
14831 
14832   ins_pipe(pipe_class_memory);
14833 
14834 %}
14835 
14836 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14837 
14838   match(Set dst (MoveF2I src));
14839 
14840   effect(DEF dst, USE src);
14841 
14842   ins_cost(INSN_COST);
14843 
14844   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14845 
14846   ins_encode %{
14847     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14848   %}
14849 
14850   ins_pipe(pipe_class_memory);
14851 
14852 %}
14853 
14854 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14855 
14856   match(Set dst (MoveI2F src));
14857 
14858   effect(DEF dst, USE src);
14859 
14860   ins_cost(INSN_COST);
14861 
14862   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14863 
14864   ins_encode %{
14865     __ strw($src$$Register, Address(sp, $dst$$disp));
14866   %}
14867 
14868   ins_pipe(istore_reg_reg);
14869 
14870 %}
14871 
14872 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14873 
14874   match(Set dst (MoveD2L src));
14875 
14876   effect(DEF dst, USE src);
14877 
14878   ins_cost(INSN_COST);
14879 
14880   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14881 
14882   ins_encode %{
14883     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14884   %}
14885 
14886   ins_pipe(pipe_class_memory);
14887 
14888 %}
14889 
14890 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14891 
14892   match(Set dst (MoveL2D src));
14893 
14894   effect(DEF dst, USE src);
14895 
14896   ins_cost(INSN_COST);
14897 
14898   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14899 
14900   ins_encode %{
14901     __ str($src$$Register, Address(sp, $dst$$disp));
14902   %}
14903 
14904   ins_pipe(istore_reg_reg);
14905 
14906 %}
14907 
14908 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14909 
14910   match(Set dst (MoveF2I src));
14911 
14912   effect(DEF dst, USE src);
14913 
14914   ins_cost(INSN_COST);
14915 
14916   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14917 
14918   ins_encode %{
14919     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14920   %}
14921 
14922   ins_pipe(fp_f2i);
14923 
14924 %}
14925 
14926 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14927 
14928   match(Set dst (MoveI2F src));
14929 
14930   effect(DEF dst, USE src);
14931 
14932   ins_cost(INSN_COST);
14933 
14934   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14935 
14936   ins_encode %{
14937     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14938   %}
14939 
14940   ins_pipe(fp_i2f);
14941 
14942 %}
14943 
14944 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14945 
14946   match(Set dst (MoveD2L src));
14947 
14948   effect(DEF dst, USE src);
14949 
14950   ins_cost(INSN_COST);
14951 
14952   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14953 
14954   ins_encode %{
14955     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14956   %}
14957 
14958   ins_pipe(fp_d2l);
14959 
14960 %}
14961 
14962 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14963 
14964   match(Set dst (MoveL2D src));
14965 
14966   effect(DEF dst, USE src);
14967 
14968   ins_cost(INSN_COST);
14969 
14970   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14971 
14972   ins_encode %{
14973     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14974   %}
14975 
14976   ins_pipe(fp_l2d);
14977 
14978 %}
14979 
14980 // ============================================================================
14981 // clearing of an array
14982 
14983 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14984 %{
14985   match(Set dummy (ClearArray cnt base));
14986   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14987 
14988   ins_cost(4 * INSN_COST);
14989   format %{ "ClearArray $cnt, $base" %}
14990 
14991   ins_encode %{
14992     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14993     if (tpc == NULL) {
14994       ciEnv::current()->record_failure("CodeCache is full");
14995       return;
14996     }
14997   %}
14998 
14999   ins_pipe(pipe_class_memory);
15000 %}
15001 
15002 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15003 %{
15004   predicate((uint64_t)n->in(2)->get_long()
15005             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15006   match(Set dummy (ClearArray cnt base));
15007   effect(TEMP temp, USE_KILL base, KILL cr);
15008 
15009   ins_cost(4 * INSN_COST);
15010   format %{ "ClearArray $cnt, $base" %}
15011 
15012   ins_encode %{
15013     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15014   %}
15015 
15016   ins_pipe(pipe_class_memory);
15017 %}
15018 
15019 // ============================================================================
15020 // Overflow Math Instructions
15021 
15022 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15023 %{
15024   match(Set cr (OverflowAddI op1 op2));
15025 
15026   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15027   ins_cost(INSN_COST);
15028   ins_encode %{
15029     __ cmnw($op1$$Register, $op2$$Register);
15030   %}
15031 
15032   ins_pipe(icmp_reg_reg);
15033 %}
15034 
15035 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15036 %{
15037   match(Set cr (OverflowAddI op1 op2));
15038 
15039   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15040   ins_cost(INSN_COST);
15041   ins_encode %{
15042     __ cmnw($op1$$Register, $op2$$constant);
15043   %}
15044 
15045   ins_pipe(icmp_reg_imm);
15046 %}
15047 
15048 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15049 %{
15050   match(Set cr (OverflowAddL op1 op2));
15051 
15052   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15053   ins_cost(INSN_COST);
15054   ins_encode %{
15055     __ cmn($op1$$Register, $op2$$Register);
15056   %}
15057 
15058   ins_pipe(icmp_reg_reg);
15059 %}
15060 
15061 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15062 %{
15063   match(Set cr (OverflowAddL op1 op2));
15064 
15065   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15066   ins_cost(INSN_COST);
15067   ins_encode %{
15068     __ cmn($op1$$Register, $op2$$constant);
15069   %}
15070 
15071   ins_pipe(icmp_reg_imm);
15072 %}
15073 
15074 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15075 %{
15076   match(Set cr (OverflowSubI op1 op2));
15077 
15078   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15079   ins_cost(INSN_COST);
15080   ins_encode %{
15081     __ cmpw($op1$$Register, $op2$$Register);
15082   %}
15083 
15084   ins_pipe(icmp_reg_reg);
15085 %}
15086 
15087 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15088 %{
15089   match(Set cr (OverflowSubI op1 op2));
15090 
15091   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15092   ins_cost(INSN_COST);
15093   ins_encode %{
15094     __ cmpw($op1$$Register, $op2$$constant);
15095   %}
15096 
15097   ins_pipe(icmp_reg_imm);
15098 %}
15099 
15100 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15101 %{
15102   match(Set cr (OverflowSubL op1 op2));
15103 
15104   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15105   ins_cost(INSN_COST);
15106   ins_encode %{
15107     __ cmp($op1$$Register, $op2$$Register);
15108   %}
15109 
15110   ins_pipe(icmp_reg_reg);
15111 %}
15112 
15113 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15114 %{
15115   match(Set cr (OverflowSubL op1 op2));
15116 
15117   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15118   ins_cost(INSN_COST);
15119   ins_encode %{
15120     __ subs(zr, $op1$$Register, $op2$$constant);
15121   %}
15122 
15123   ins_pipe(icmp_reg_imm);
15124 %}
15125 
15126 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15127 %{
15128   match(Set cr (OverflowSubI zero op1));
15129 
15130   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15131   ins_cost(INSN_COST);
15132   ins_encode %{
15133     __ cmpw(zr, $op1$$Register);
15134   %}
15135 
15136   ins_pipe(icmp_reg_imm);
15137 %}
15138 
15139 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15140 %{
15141   match(Set cr (OverflowSubL zero op1));
15142 
15143   format %{ "cmp   zr, $op1\t# overflow check long" %}
15144   ins_cost(INSN_COST);
15145   ins_encode %{
15146     __ cmp(zr, $op1$$Register);
15147   %}
15148 
15149   ins_pipe(icmp_reg_imm);
15150 %}
15151 
15152 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15153 %{
15154   match(Set cr (OverflowMulI op1 op2));
15155 
15156   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15157             "cmp   rscratch1, rscratch1, sxtw\n\t"
15158             "movw  rscratch1, #0x80000000\n\t"
15159             "cselw rscratch1, rscratch1, zr, NE\n\t"
15160             "cmpw  rscratch1, #1" %}
15161   ins_cost(5 * INSN_COST);
15162   ins_encode %{
15163     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15164     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15165     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15166     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15167     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15168   %}
15169 
15170   ins_pipe(pipe_slow);
15171 %}
15172 
15173 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15174 %{
15175   match(If cmp (OverflowMulI op1 op2));
15176   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15177             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15178   effect(USE labl, KILL cr);
15179 
15180   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15181             "cmp   rscratch1, rscratch1, sxtw\n\t"
15182             "b$cmp   $labl" %}
15183   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15184   ins_encode %{
15185     Label* L = $labl$$label;
15186     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15187     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15188     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15189     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15190   %}
15191 
15192   ins_pipe(pipe_serial);
15193 %}
15194 
15195 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15196 %{
15197   match(Set cr (OverflowMulL op1 op2));
15198 
15199   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15200             "smulh rscratch2, $op1, $op2\n\t"
15201             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15202             "movw  rscratch1, #0x80000000\n\t"
15203             "cselw rscratch1, rscratch1, zr, NE\n\t"
15204             "cmpw  rscratch1, #1" %}
15205   ins_cost(6 * INSN_COST);
15206   ins_encode %{
15207     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15208     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15209     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15210     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15211     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15212     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15213   %}
15214 
15215   ins_pipe(pipe_slow);
15216 %}
15217 
15218 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15219 %{
15220   match(If cmp (OverflowMulL op1 op2));
15221   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15222             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15223   effect(USE labl, KILL cr);
15224 
15225   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15226             "smulh rscratch2, $op1, $op2\n\t"
15227             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15228             "b$cmp $labl" %}
15229   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15230   ins_encode %{
15231     Label* L = $labl$$label;
15232     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15233     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15234     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15235     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15236     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15237   %}
15238 
15239   ins_pipe(pipe_serial);
15240 %}
15241 
15242 // ============================================================================
15243 // Compare Instructions
15244 
15245 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15246 %{
15247   match(Set cr (CmpI op1 op2));
15248 
15249   effect(DEF cr, USE op1, USE op2);
15250 
15251   ins_cost(INSN_COST);
15252   format %{ "cmpw  $op1, $op2" %}
15253 
15254   ins_encode(aarch64_enc_cmpw(op1, op2));
15255 
15256   ins_pipe(icmp_reg_reg);
15257 %}
15258 
15259 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15260 %{
15261   match(Set cr (CmpI op1 zero));
15262 
15263   effect(DEF cr, USE op1);
15264 
15265   ins_cost(INSN_COST);
15266   format %{ "cmpw $op1, 0" %}
15267 
15268   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15269 
15270   ins_pipe(icmp_reg_imm);
15271 %}
15272 
15273 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15274 %{
15275   match(Set cr (CmpI op1 op2));
15276 
15277   effect(DEF cr, USE op1);
15278 
15279   ins_cost(INSN_COST);
15280   format %{ "cmpw  $op1, $op2" %}
15281 
15282   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15283 
15284   ins_pipe(icmp_reg_imm);
15285 %}
15286 
15287 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15288 %{
15289   match(Set cr (CmpI op1 op2));
15290 
15291   effect(DEF cr, USE op1);
15292 
15293   ins_cost(INSN_COST * 2);
15294   format %{ "cmpw  $op1, $op2" %}
15295 
15296   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15297 
15298   ins_pipe(icmp_reg_imm);
15299 %}
15300 
15301 // Unsigned compare Instructions; really, same as signed compare
15302 // except it should only be used to feed an If or a CMovI which takes a
15303 // cmpOpU.
15304 
15305 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15306 %{
15307   match(Set cr (CmpU op1 op2));
15308 
15309   effect(DEF cr, USE op1, USE op2);
15310 
15311   ins_cost(INSN_COST);
15312   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15313 
15314   ins_encode(aarch64_enc_cmpw(op1, op2));
15315 
15316   ins_pipe(icmp_reg_reg);
15317 %}
15318 
15319 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15320 %{
15321   match(Set cr (CmpU op1 zero));
15322 
15323   effect(DEF cr, USE op1);
15324 
15325   ins_cost(INSN_COST);
15326   format %{ "cmpw $op1, #0\t# unsigned" %}
15327 
15328   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15329 
15330   ins_pipe(icmp_reg_imm);
15331 %}
15332 
15333 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15334 %{
15335   match(Set cr (CmpU op1 op2));
15336 
15337   effect(DEF cr, USE op1);
15338 
15339   ins_cost(INSN_COST);
15340   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15341 
15342   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15343 
15344   ins_pipe(icmp_reg_imm);
15345 %}
15346 
15347 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15348 %{
15349   match(Set cr (CmpU op1 op2));
15350 
15351   effect(DEF cr, USE op1);
15352 
15353   ins_cost(INSN_COST * 2);
15354   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15355 
15356   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15357 
15358   ins_pipe(icmp_reg_imm);
15359 %}
15360 
15361 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15362 %{
15363   match(Set cr (CmpL op1 op2));
15364 
15365   effect(DEF cr, USE op1, USE op2);
15366 
15367   ins_cost(INSN_COST);
15368   format %{ "cmp  $op1, $op2" %}
15369 
15370   ins_encode(aarch64_enc_cmp(op1, op2));
15371 
15372   ins_pipe(icmp_reg_reg);
15373 %}
15374 
15375 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15376 %{
15377   match(Set cr (CmpL op1 zero));
15378 
15379   effect(DEF cr, USE op1);
15380 
15381   ins_cost(INSN_COST);
15382   format %{ "tst  $op1" %}
15383 
15384   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15385 
15386   ins_pipe(icmp_reg_imm);
15387 %}
15388 
15389 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15390 %{
15391   match(Set cr (CmpL op1 op2));
15392 
15393   effect(DEF cr, USE op1);
15394 
15395   ins_cost(INSN_COST);
15396   format %{ "cmp  $op1, $op2" %}
15397 
15398   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15399 
15400   ins_pipe(icmp_reg_imm);
15401 %}
15402 
15403 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15404 %{
15405   match(Set cr (CmpL op1 op2));
15406 
15407   effect(DEF cr, USE op1);
15408 
15409   ins_cost(INSN_COST * 2);
15410   format %{ "cmp  $op1, $op2" %}
15411 
15412   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15413 
15414   ins_pipe(icmp_reg_imm);
15415 %}
15416 
15417 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15418 %{
15419   match(Set cr (CmpUL op1 op2));
15420 
15421   effect(DEF cr, USE op1, USE op2);
15422 
15423   ins_cost(INSN_COST);
15424   format %{ "cmp  $op1, $op2" %}
15425 
15426   ins_encode(aarch64_enc_cmp(op1, op2));
15427 
15428   ins_pipe(icmp_reg_reg);
15429 %}
15430 
15431 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15432 %{
15433   match(Set cr (CmpUL op1 zero));
15434 
15435   effect(DEF cr, USE op1);
15436 
15437   ins_cost(INSN_COST);
15438   format %{ "tst  $op1" %}
15439 
15440   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15441 
15442   ins_pipe(icmp_reg_imm);
15443 %}
15444 
15445 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15446 %{
15447   match(Set cr (CmpUL op1 op2));
15448 
15449   effect(DEF cr, USE op1);
15450 
15451   ins_cost(INSN_COST);
15452   format %{ "cmp  $op1, $op2" %}
15453 
15454   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15455 
15456   ins_pipe(icmp_reg_imm);
15457 %}
15458 
15459 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15460 %{
15461   match(Set cr (CmpUL op1 op2));
15462 
15463   effect(DEF cr, USE op1);
15464 
15465   ins_cost(INSN_COST * 2);
15466   format %{ "cmp  $op1, $op2" %}
15467 
15468   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15469 
15470   ins_pipe(icmp_reg_imm);
15471 %}
15472 
15473 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15474 %{
15475   match(Set cr (CmpP op1 op2));
15476 
15477   effect(DEF cr, USE op1, USE op2);
15478 
15479   ins_cost(INSN_COST);
15480   format %{ "cmp  $op1, $op2\t // ptr" %}
15481 
15482   ins_encode(aarch64_enc_cmpp(op1, op2));
15483 
15484   ins_pipe(icmp_reg_reg);
15485 %}
15486 
15487 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15488 %{
15489   match(Set cr (CmpN op1 op2));
15490 
15491   effect(DEF cr, USE op1, USE op2);
15492 
15493   ins_cost(INSN_COST);
15494   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15495 
15496   ins_encode(aarch64_enc_cmpn(op1, op2));
15497 
15498   ins_pipe(icmp_reg_reg);
15499 %}
15500 
15501 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15502 %{
15503   match(Set cr (CmpP op1 zero));
15504 
15505   effect(DEF cr, USE op1, USE zero);
15506 
15507   ins_cost(INSN_COST);
15508   format %{ "cmp  $op1, 0\t // ptr" %}
15509 
15510   ins_encode(aarch64_enc_testp(op1));
15511 
15512   ins_pipe(icmp_reg_imm);
15513 %}
15514 
15515 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15516 %{
15517   match(Set cr (CmpN op1 zero));
15518 
15519   effect(DEF cr, USE op1, USE zero);
15520 
15521   ins_cost(INSN_COST);
15522   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15523 
15524   ins_encode(aarch64_enc_testn(op1));
15525 
15526   ins_pipe(icmp_reg_imm);
15527 %}
15528 
15529 // FP comparisons
15530 //
15531 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15532 // using normal cmpOp. See declaration of rFlagsReg for details.
15533 
15534 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15535 %{
15536   match(Set cr (CmpF src1 src2));
15537 
15538   ins_cost(3 * INSN_COST);
15539   format %{ "fcmps $src1, $src2" %}
15540 
15541   ins_encode %{
15542     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15543   %}
15544 
15545   ins_pipe(pipe_class_compare);
15546 %}
15547 
15548 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15549 %{
15550   match(Set cr (CmpF src1 src2));
15551 
15552   ins_cost(3 * INSN_COST);
15553   format %{ "fcmps $src1, 0.0" %}
15554 
15555   ins_encode %{
15556     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15557   %}
15558 
15559   ins_pipe(pipe_class_compare);
15560 %}
15561 // FROM HERE
15562 
15563 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15564 %{
15565   match(Set cr (CmpD src1 src2));
15566 
15567   ins_cost(3 * INSN_COST);
15568   format %{ "fcmpd $src1, $src2" %}
15569 
15570   ins_encode %{
15571     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15572   %}
15573 
15574   ins_pipe(pipe_class_compare);
15575 %}
15576 
15577 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15578 %{
15579   match(Set cr (CmpD src1 src2));
15580 
15581   ins_cost(3 * INSN_COST);
15582   format %{ "fcmpd $src1, 0.0" %}
15583 
15584   ins_encode %{
15585     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15586   %}
15587 
15588   ins_pipe(pipe_class_compare);
15589 %}
15590 
15591 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15592 %{
15593   match(Set dst (CmpF3 src1 src2));
15594   effect(KILL cr);
15595 
15596   ins_cost(5 * INSN_COST);
15597   format %{ "fcmps $src1, $src2\n\t"
15598             "csinvw($dst, zr, zr, eq\n\t"
15599             "csnegw($dst, $dst, $dst, lt)"
15600   %}
15601 
15602   ins_encode %{
15603     Label done;
15604     FloatRegister s1 = as_FloatRegister($src1$$reg);
15605     FloatRegister s2 = as_FloatRegister($src2$$reg);
15606     Register d = as_Register($dst$$reg);
15607     __ fcmps(s1, s2);
15608     // installs 0 if EQ else -1
15609     __ csinvw(d, zr, zr, Assembler::EQ);
15610     // keeps -1 if less or unordered else installs 1
15611     __ csnegw(d, d, d, Assembler::LT);
15612     __ bind(done);
15613   %}
15614 
15615   ins_pipe(pipe_class_default);
15616 
15617 %}
15618 
15619 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15620 %{
15621   match(Set dst (CmpD3 src1 src2));
15622   effect(KILL cr);
15623 
15624   ins_cost(5 * INSN_COST);
15625   format %{ "fcmpd $src1, $src2\n\t"
15626             "csinvw($dst, zr, zr, eq\n\t"
15627             "csnegw($dst, $dst, $dst, lt)"
15628   %}
15629 
15630   ins_encode %{
15631     Label done;
15632     FloatRegister s1 = as_FloatRegister($src1$$reg);
15633     FloatRegister s2 = as_FloatRegister($src2$$reg);
15634     Register d = as_Register($dst$$reg);
15635     __ fcmpd(s1, s2);
15636     // installs 0 if EQ else -1
15637     __ csinvw(d, zr, zr, Assembler::EQ);
15638     // keeps -1 if less or unordered else installs 1
15639     __ csnegw(d, d, d, Assembler::LT);
15640     __ bind(done);
15641   %}
15642   ins_pipe(pipe_class_default);
15643 
15644 %}
15645 
15646 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15647 %{
15648   match(Set dst (CmpF3 src1 zero));
15649   effect(KILL cr);
15650 
15651   ins_cost(5 * INSN_COST);
15652   format %{ "fcmps $src1, 0.0\n\t"
15653             "csinvw($dst, zr, zr, eq\n\t"
15654             "csnegw($dst, $dst, $dst, lt)"
15655   %}
15656 
15657   ins_encode %{
15658     Label done;
15659     FloatRegister s1 = as_FloatRegister($src1$$reg);
15660     Register d = as_Register($dst$$reg);
15661     __ fcmps(s1, 0.0);
15662     // installs 0 if EQ else -1
15663     __ csinvw(d, zr, zr, Assembler::EQ);
15664     // keeps -1 if less or unordered else installs 1
15665     __ csnegw(d, d, d, Assembler::LT);
15666     __ bind(done);
15667   %}
15668 
15669   ins_pipe(pipe_class_default);
15670 
15671 %}
15672 
15673 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15674 %{
15675   match(Set dst (CmpD3 src1 zero));
15676   effect(KILL cr);
15677 
15678   ins_cost(5 * INSN_COST);
15679   format %{ "fcmpd $src1, 0.0\n\t"
15680             "csinvw($dst, zr, zr, eq\n\t"
15681             "csnegw($dst, $dst, $dst, lt)"
15682   %}
15683 
15684   ins_encode %{
15685     Label done;
15686     FloatRegister s1 = as_FloatRegister($src1$$reg);
15687     Register d = as_Register($dst$$reg);
15688     __ fcmpd(s1, 0.0);
15689     // installs 0 if EQ else -1
15690     __ csinvw(d, zr, zr, Assembler::EQ);
15691     // keeps -1 if less or unordered else installs 1
15692     __ csnegw(d, d, d, Assembler::LT);
15693     __ bind(done);
15694   %}
15695   ins_pipe(pipe_class_default);
15696 
15697 %}
15698 
15699 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15700 %{
15701   match(Set dst (CmpLTMask p q));
15702   effect(KILL cr);
15703 
15704   ins_cost(3 * INSN_COST);
15705 
15706   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15707             "csetw $dst, lt\n\t"
15708             "subw $dst, zr, $dst"
15709   %}
15710 
15711   ins_encode %{
15712     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15713     __ csetw(as_Register($dst$$reg), Assembler::LT);
15714     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15715   %}
15716 
15717   ins_pipe(ialu_reg_reg);
15718 %}
15719 
15720 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15721 %{
15722   match(Set dst (CmpLTMask src zero));
15723   effect(KILL cr);
15724 
15725   ins_cost(INSN_COST);
15726 
15727   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15728 
15729   ins_encode %{
15730     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15731   %}
15732 
15733   ins_pipe(ialu_reg_shift);
15734 %}
15735 
15736 // ============================================================================
15737 // Max and Min
15738 
15739 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15740 %{
15741   effect( DEF dst, USE src1, USE src2, USE cr );
15742 
15743   ins_cost(INSN_COST * 2);
15744   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15745 
15746   ins_encode %{
15747     __ cselw(as_Register($dst$$reg),
15748              as_Register($src1$$reg),
15749              as_Register($src2$$reg),
15750              Assembler::LT);
15751   %}
15752 
15753   ins_pipe(icond_reg_reg);
15754 %}
15755 
15756 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15757 %{
15758   match(Set dst (MinI src1 src2));
15759   ins_cost(INSN_COST * 3);
15760 
15761   expand %{
15762     rFlagsReg cr;
15763     compI_reg_reg(cr, src1, src2);
15764     cmovI_reg_reg_lt(dst, src1, src2, cr);
15765   %}
15766 
15767 %}
15768 // FROM HERE
15769 
15770 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15771 %{
15772   effect( DEF dst, USE src1, USE src2, USE cr );
15773 
15774   ins_cost(INSN_COST * 2);
15775   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15776 
15777   ins_encode %{
15778     __ cselw(as_Register($dst$$reg),
15779              as_Register($src1$$reg),
15780              as_Register($src2$$reg),
15781              Assembler::GT);
15782   %}
15783 
15784   ins_pipe(icond_reg_reg);
15785 %}
15786 
15787 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15788 %{
15789   match(Set dst (MaxI src1 src2));
15790   ins_cost(INSN_COST * 3);
15791   expand %{
15792     rFlagsReg cr;
15793     compI_reg_reg(cr, src1, src2);
15794     cmovI_reg_reg_gt(dst, src1, src2, cr);
15795   %}
15796 %}
15797 
15798 // ============================================================================
15799 // Branch Instructions
15800 
15801 // Direct Branch.
15802 instruct branch(label lbl)
15803 %{
15804   match(Goto);
15805 
15806   effect(USE lbl);
15807 
15808   ins_cost(BRANCH_COST);
15809   format %{ "b  $lbl" %}
15810 
15811   ins_encode(aarch64_enc_b(lbl));
15812 
15813   ins_pipe(pipe_branch);
15814 %}
15815 
15816 // Conditional Near Branch
15817 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15818 %{
15819   // Same match rule as `branchConFar'.
15820   match(If cmp cr);
15821 
15822   effect(USE lbl);
15823 
15824   ins_cost(BRANCH_COST);
15825   // If set to 1 this indicates that the current instruction is a
15826   // short variant of a long branch. This avoids using this
15827   // instruction in first-pass matching. It will then only be used in
15828   // the `Shorten_branches' pass.
15829   // ins_short_branch(1);
15830   format %{ "b$cmp  $lbl" %}
15831 
15832   ins_encode(aarch64_enc_br_con(cmp, lbl));
15833 
15834   ins_pipe(pipe_branch_cond);
15835 %}
15836 
15837 // Conditional Near Branch Unsigned
15838 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15839 %{
15840   // Same match rule as `branchConFar'.
15841   match(If cmp cr);
15842 
15843   effect(USE lbl);
15844 
15845   ins_cost(BRANCH_COST);
15846   // If set to 1 this indicates that the current instruction is a
15847   // short variant of a long branch. This avoids using this
15848   // instruction in first-pass matching. It will then only be used in
15849   // the `Shorten_branches' pass.
15850   // ins_short_branch(1);
15851   format %{ "b$cmp  $lbl\t# unsigned" %}
15852 
15853   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15854 
15855   ins_pipe(pipe_branch_cond);
15856 %}
15857 
15858 // Make use of CBZ and CBNZ.  These instructions, as well as being
15859 // shorter than (cmp; branch), have the additional benefit of not
15860 // killing the flags.
15861 
15862 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15863   match(If cmp (CmpI op1 op2));
15864   effect(USE labl);
15865 
15866   ins_cost(BRANCH_COST);
15867   format %{ "cbw$cmp   $op1, $labl" %}
15868   ins_encode %{
15869     Label* L = $labl$$label;
15870     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15871     if (cond == Assembler::EQ)
15872       __ cbzw($op1$$Register, *L);
15873     else
15874       __ cbnzw($op1$$Register, *L);
15875   %}
15876   ins_pipe(pipe_cmp_branch);
15877 %}
15878 
15879 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15880   match(If cmp (CmpL op1 op2));
15881   effect(USE labl);
15882 
15883   ins_cost(BRANCH_COST);
15884   format %{ "cb$cmp   $op1, $labl" %}
15885   ins_encode %{
15886     Label* L = $labl$$label;
15887     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15888     if (cond == Assembler::EQ)
15889       __ cbz($op1$$Register, *L);
15890     else
15891       __ cbnz($op1$$Register, *L);
15892   %}
15893   ins_pipe(pipe_cmp_branch);
15894 %}
15895 
15896 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15897   match(If cmp (CmpP op1 op2));
15898   effect(USE labl);
15899 
15900   ins_cost(BRANCH_COST);
15901   format %{ "cb$cmp   $op1, $labl" %}
15902   ins_encode %{
15903     Label* L = $labl$$label;
15904     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15905     if (cond == Assembler::EQ)
15906       __ cbz($op1$$Register, *L);
15907     else
15908       __ cbnz($op1$$Register, *L);
15909   %}
15910   ins_pipe(pipe_cmp_branch);
15911 %}
15912 
15913 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15914   match(If cmp (CmpN op1 op2));
15915   effect(USE labl);
15916 
15917   ins_cost(BRANCH_COST);
15918   format %{ "cbw$cmp   $op1, $labl" %}
15919   ins_encode %{
15920     Label* L = $labl$$label;
15921     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15922     if (cond == Assembler::EQ)
15923       __ cbzw($op1$$Register, *L);
15924     else
15925       __ cbnzw($op1$$Register, *L);
15926   %}
15927   ins_pipe(pipe_cmp_branch);
15928 %}
15929 
15930 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15931   match(If cmp (CmpP (DecodeN oop) zero));
15932   effect(USE labl);
15933 
15934   ins_cost(BRANCH_COST);
15935   format %{ "cb$cmp   $oop, $labl" %}
15936   ins_encode %{
15937     Label* L = $labl$$label;
15938     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15939     if (cond == Assembler::EQ)
15940       __ cbzw($oop$$Register, *L);
15941     else
15942       __ cbnzw($oop$$Register, *L);
15943   %}
15944   ins_pipe(pipe_cmp_branch);
15945 %}
15946 
15947 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15948   match(If cmp (CmpU op1 op2));
15949   effect(USE labl);
15950 
15951   ins_cost(BRANCH_COST);
15952   format %{ "cbw$cmp   $op1, $labl" %}
15953   ins_encode %{
15954     Label* L = $labl$$label;
15955     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15956     if (cond == Assembler::EQ || cond == Assembler::LS)
15957       __ cbzw($op1$$Register, *L);
15958     else
15959       __ cbnzw($op1$$Register, *L);
15960   %}
15961   ins_pipe(pipe_cmp_branch);
15962 %}
15963 
15964 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15965   match(If cmp (CmpUL op1 op2));
15966   effect(USE labl);
15967 
15968   ins_cost(BRANCH_COST);
15969   format %{ "cb$cmp   $op1, $labl" %}
15970   ins_encode %{
15971     Label* L = $labl$$label;
15972     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15973     if (cond == Assembler::EQ || cond == Assembler::LS)
15974       __ cbz($op1$$Register, *L);
15975     else
15976       __ cbnz($op1$$Register, *L);
15977   %}
15978   ins_pipe(pipe_cmp_branch);
15979 %}
15980 
15981 // Test bit and Branch
15982 
15983 // Patterns for short (< 32KiB) variants
15984 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15985   match(If cmp (CmpL op1 op2));
15986   effect(USE labl);
15987 
15988   ins_cost(BRANCH_COST);
15989   format %{ "cb$cmp   $op1, $labl # long" %}
15990   ins_encode %{
15991     Label* L = $labl$$label;
15992     Assembler::Condition cond =
15993       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15994     __ tbr(cond, $op1$$Register, 63, *L);
15995   %}
15996   ins_pipe(pipe_cmp_branch);
15997   ins_short_branch(1);
15998 %}
15999 
16000 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16001   match(If cmp (CmpI op1 op2));
16002   effect(USE labl);
16003 
16004   ins_cost(BRANCH_COST);
16005   format %{ "cb$cmp   $op1, $labl # int" %}
16006   ins_encode %{
16007     Label* L = $labl$$label;
16008     Assembler::Condition cond =
16009       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16010     __ tbr(cond, $op1$$Register, 31, *L);
16011   %}
16012   ins_pipe(pipe_cmp_branch);
16013   ins_short_branch(1);
16014 %}
16015 
16016 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16017   match(If cmp (CmpL (AndL op1 op2) op3));
16018   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16019   effect(USE labl);
16020 
16021   ins_cost(BRANCH_COST);
16022   format %{ "tb$cmp   $op1, $op2, $labl" %}
16023   ins_encode %{
16024     Label* L = $labl$$label;
16025     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16026     int bit = exact_log2_long($op2$$constant);
16027     __ tbr(cond, $op1$$Register, bit, *L);
16028   %}
16029   ins_pipe(pipe_cmp_branch);
16030   ins_short_branch(1);
16031 %}
16032 
16033 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16034   match(If cmp (CmpI (AndI op1 op2) op3));
16035   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16036   effect(USE labl);
16037 
16038   ins_cost(BRANCH_COST);
16039   format %{ "tb$cmp   $op1, $op2, $labl" %}
16040   ins_encode %{
16041     Label* L = $labl$$label;
16042     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16043     int bit = exact_log2((juint)$op2$$constant);
16044     __ tbr(cond, $op1$$Register, bit, *L);
16045   %}
16046   ins_pipe(pipe_cmp_branch);
16047   ins_short_branch(1);
16048 %}
16049 
16050 // And far variants
16051 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16052   match(If cmp (CmpL op1 op2));
16053   effect(USE labl);
16054 
16055   ins_cost(BRANCH_COST);
16056   format %{ "cb$cmp   $op1, $labl # long" %}
16057   ins_encode %{
16058     Label* L = $labl$$label;
16059     Assembler::Condition cond =
16060       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16061     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16062   %}
16063   ins_pipe(pipe_cmp_branch);
16064 %}
16065 
16066 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16067   match(If cmp (CmpI op1 op2));
16068   effect(USE labl);
16069 
16070   ins_cost(BRANCH_COST);
16071   format %{ "cb$cmp   $op1, $labl # int" %}
16072   ins_encode %{
16073     Label* L = $labl$$label;
16074     Assembler::Condition cond =
16075       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16076     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16077   %}
16078   ins_pipe(pipe_cmp_branch);
16079 %}
16080 
16081 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16082   match(If cmp (CmpL (AndL op1 op2) op3));
16083   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16084   effect(USE labl);
16085 
16086   ins_cost(BRANCH_COST);
16087   format %{ "tb$cmp   $op1, $op2, $labl" %}
16088   ins_encode %{
16089     Label* L = $labl$$label;
16090     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16091     int bit = exact_log2_long($op2$$constant);
16092     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16093   %}
16094   ins_pipe(pipe_cmp_branch);
16095 %}
16096 
16097 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16098   match(If cmp (CmpI (AndI op1 op2) op3));
16099   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16100   effect(USE labl);
16101 
16102   ins_cost(BRANCH_COST);
16103   format %{ "tb$cmp   $op1, $op2, $labl" %}
16104   ins_encode %{
16105     Label* L = $labl$$label;
16106     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16107     int bit = exact_log2((juint)$op2$$constant);
16108     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16109   %}
16110   ins_pipe(pipe_cmp_branch);
16111 %}
16112 
16113 // Test bits
16114 
16115 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16116   match(Set cr (CmpL (AndL op1 op2) op3));
16117   predicate(Assembler::operand_valid_for_logical_immediate
16118             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16119 
16120   ins_cost(INSN_COST);
16121   format %{ "tst $op1, $op2 # long" %}
16122   ins_encode %{
16123     __ tst($op1$$Register, $op2$$constant);
16124   %}
16125   ins_pipe(ialu_reg_reg);
16126 %}
16127 
16128 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16129   match(Set cr (CmpI (AndI op1 op2) op3));
16130   predicate(Assembler::operand_valid_for_logical_immediate
16131             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16132 
16133   ins_cost(INSN_COST);
16134   format %{ "tst $op1, $op2 # int" %}
16135   ins_encode %{
16136     __ tstw($op1$$Register, $op2$$constant);
16137   %}
16138   ins_pipe(ialu_reg_reg);
16139 %}
16140 
16141 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16142   match(Set cr (CmpL (AndL op1 op2) op3));
16143 
16144   ins_cost(INSN_COST);
16145   format %{ "tst $op1, $op2 # long" %}
16146   ins_encode %{
16147     __ tst($op1$$Register, $op2$$Register);
16148   %}
16149   ins_pipe(ialu_reg_reg);
16150 %}
16151 
16152 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16153   match(Set cr (CmpI (AndI op1 op2) op3));
16154 
16155   ins_cost(INSN_COST);
16156   format %{ "tstw $op1, $op2 # int" %}
16157   ins_encode %{
16158     __ tstw($op1$$Register, $op2$$Register);
16159   %}
16160   ins_pipe(ialu_reg_reg);
16161 %}
16162 
16163 
16164 // Conditional Far Branch
16165 // Conditional Far Branch Unsigned
16166 // TODO: fixme
16167 
16168 // counted loop end branch near
16169 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16170 %{
16171   match(CountedLoopEnd cmp cr);
16172 
16173   effect(USE lbl);
16174 
16175   ins_cost(BRANCH_COST);
16176   // short variant.
16177   // ins_short_branch(1);
16178   format %{ "b$cmp $lbl \t// counted loop end" %}
16179 
16180   ins_encode(aarch64_enc_br_con(cmp, lbl));
16181 
16182   ins_pipe(pipe_branch);
16183 %}
16184 
16185 // counted loop end branch near Unsigned
16186 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16187 %{
16188   match(CountedLoopEnd cmp cr);
16189 
16190   effect(USE lbl);
16191 
16192   ins_cost(BRANCH_COST);
16193   // short variant.
16194   // ins_short_branch(1);
16195   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16196 
16197   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16198 
16199   ins_pipe(pipe_branch);
16200 %}
16201 
16202 // counted loop end branch far
16203 // counted loop end branch far unsigned
16204 // TODO: fixme
16205 
16206 // ============================================================================
16207 // inlined locking and unlocking
16208 
16209 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16210 %{
16211   match(Set cr (FastLock object box));
16212   effect(TEMP tmp, TEMP tmp2);
16213 
16214   // TODO
16215   // identify correct cost
16216   ins_cost(5 * INSN_COST);
16217   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16218 
16219   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16220 
16221   ins_pipe(pipe_serial);
16222 %}
16223 
16224 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16225 %{
16226   match(Set cr (FastUnlock object box));
16227   effect(TEMP tmp, TEMP tmp2);
16228 
16229   ins_cost(5 * INSN_COST);
16230   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16231 
16232   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16233 
16234   ins_pipe(pipe_serial);
16235 %}
16236 
16237 
16238 // ============================================================================
16239 // Safepoint Instructions
16240 
16241 // TODO
16242 // provide a near and far version of this code
16243 
16244 instruct safePoint(rFlagsReg cr, iRegP poll)
16245 %{
16246   match(SafePoint poll);
16247   effect(KILL cr);
16248 
16249   format %{
16250     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16251   %}
16252   ins_encode %{
16253     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16254   %}
16255   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16256 %}
16257 
16258 
16259 // ============================================================================
16260 // Procedure Call/Return Instructions
16261 
16262 // Call Java Static Instruction
16263 
16264 instruct CallStaticJavaDirect(method meth)
16265 %{
16266   match(CallStaticJava);
16267 
16268   effect(USE meth);
16269 
16270   ins_cost(CALL_COST);
16271 
16272   format %{ "call,static $meth \t// ==> " %}
16273 
16274   ins_encode(aarch64_enc_java_static_call(meth),
16275              aarch64_enc_call_epilog);
16276 
16277   ins_pipe(pipe_class_call);
16278 %}
16279 
16280 // TO HERE
16281 
16282 // Call Java Dynamic Instruction
16283 instruct CallDynamicJavaDirect(method meth)
16284 %{
16285   match(CallDynamicJava);
16286 
16287   effect(USE meth);
16288 
16289   ins_cost(CALL_COST);
16290 
16291   format %{ "CALL,dynamic $meth \t// ==> " %}
16292 
16293   ins_encode(aarch64_enc_java_dynamic_call(meth),
16294              aarch64_enc_call_epilog);
16295 
16296   ins_pipe(pipe_class_call);
16297 %}
16298 
16299 // Call Runtime Instruction
16300 
16301 instruct CallRuntimeDirect(method meth)
16302 %{
16303   match(CallRuntime);
16304 
16305   effect(USE meth);
16306 
16307   ins_cost(CALL_COST);
16308 
16309   format %{ "CALL, runtime $meth" %}
16310 
16311   ins_encode( aarch64_enc_java_to_runtime(meth) );
16312 
16313   ins_pipe(pipe_class_call);
16314 %}
16315 
16316 // Call Runtime Instruction
16317 
16318 instruct CallLeafDirect(method meth)
16319 %{
16320   match(CallLeaf);
16321 
16322   effect(USE meth);
16323 
16324   ins_cost(CALL_COST);
16325 
16326   format %{ "CALL, runtime leaf $meth" %}
16327 
16328   ins_encode( aarch64_enc_java_to_runtime(meth) );
16329 
16330   ins_pipe(pipe_class_call);
16331 %}
16332 
16333 // Call Runtime Instruction
16334 
16335 instruct CallLeafNoFPDirect(method meth)
16336 %{
16337   match(CallLeafNoFP);
16338 
16339   effect(USE meth);
16340 
16341   ins_cost(CALL_COST);
16342 
16343   format %{ "CALL, runtime leaf nofp $meth" %}
16344 
16345   ins_encode( aarch64_enc_java_to_runtime(meth) );
16346 
16347   ins_pipe(pipe_class_call);
16348 %}
16349 
16350 instruct CallNativeDirect(method meth)
16351 %{
16352   match(CallNative);
16353 
16354   effect(USE meth);
16355 
16356   ins_cost(CALL_COST);
16357 
16358   format %{ "CALL, native $meth" %}
16359 
16360   ins_encode( aarch64_enc_java_to_runtime(meth) );
16361 
16362   ins_pipe(pipe_class_call);
16363 %}
16364 
16365 // Tail Call; Jump from runtime stub to Java code.
16366 // Also known as an 'interprocedural jump'.
16367 // Target of jump will eventually return to caller.
16368 // TailJump below removes the return address.
16369 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16370 %{
16371   match(TailCall jump_target method_ptr);
16372 
16373   ins_cost(CALL_COST);
16374 
16375   format %{ "br $jump_target\t# $method_ptr holds method" %}
16376 
16377   ins_encode(aarch64_enc_tail_call(jump_target));
16378 
16379   ins_pipe(pipe_class_call);
16380 %}
16381 
16382 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16383 %{
16384   match(TailJump jump_target ex_oop);
16385 
16386   ins_cost(CALL_COST);
16387 
16388   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16389 
16390   ins_encode(aarch64_enc_tail_jmp(jump_target));
16391 
16392   ins_pipe(pipe_class_call);
16393 %}
16394 
16395 // Create exception oop: created by stack-crawling runtime code.
16396 // Created exception is now available to this handler, and is setup
16397 // just prior to jumping to this handler. No code emitted.
16398 // TODO check
16399 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16400 instruct CreateException(iRegP_R0 ex_oop)
16401 %{
16402   match(Set ex_oop (CreateEx));
16403 
16404   format %{ " -- \t// exception oop; no code emitted" %}
16405 
16406   size(0);
16407 
16408   ins_encode( /*empty*/ );
16409 
16410   ins_pipe(pipe_class_empty);
16411 %}
16412 
16413 // Rethrow exception: The exception oop will come in the first
16414 // argument position. Then JUMP (not call) to the rethrow stub code.
16415 instruct RethrowException() %{
16416   match(Rethrow);
16417   ins_cost(CALL_COST);
16418 
16419   format %{ "b rethrow_stub" %}
16420 
16421   ins_encode( aarch64_enc_rethrow() );
16422 
16423   ins_pipe(pipe_class_call);
16424 %}
16425 
16426 
16427 // Return Instruction
16428 // epilog node loads ret address into lr as part of frame pop
16429 instruct Ret()
16430 %{
16431   match(Return);
16432 
16433   format %{ "ret\t// return register" %}
16434 
16435   ins_encode( aarch64_enc_ret() );
16436 
16437   ins_pipe(pipe_branch);
16438 %}
16439 
16440 // Die now.
16441 instruct ShouldNotReachHere() %{
16442   match(Halt);
16443 
16444   ins_cost(CALL_COST);
16445   format %{ "ShouldNotReachHere" %}
16446 
16447   ins_encode %{
16448     if (is_reachable()) {
16449       __ stop(_halt_reason);
16450     }
16451   %}
16452 
16453   ins_pipe(pipe_class_default);
16454 %}
16455 
16456 // ============================================================================
16457 // Partial Subtype Check
16458 //
16459 // superklass array for an instance of the superklass.  Set a hidden
16460 // internal cache on a hit (cache is checked with exposed code in
16461 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16462 // encoding ALSO sets flags.
16463 
16464 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16465 %{
16466   match(Set result (PartialSubtypeCheck sub super));
16467   effect(KILL cr, KILL temp);
16468 
16469   ins_cost(1100);  // slightly larger than the next version
16470   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16471 
16472   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16473 
16474   opcode(0x1); // Force zero of result reg on hit
16475 
16476   ins_pipe(pipe_class_memory);
16477 %}
16478 
16479 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16480 %{
16481   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16482   effect(KILL temp, KILL result);
16483 
16484   ins_cost(1100);  // slightly larger than the next version
16485   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16486 
16487   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16488 
16489   opcode(0x0); // Don't zero result reg on hit
16490 
16491   ins_pipe(pipe_class_memory);
16492 %}
16493 
16494 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16495                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16496 %{
16497   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16498   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16499   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16500 
16501   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16502   ins_encode %{
16503     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16504     __ string_compare($str1$$Register, $str2$$Register,
16505                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16506                       $tmp1$$Register, $tmp2$$Register,
16507                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16508   %}
16509   ins_pipe(pipe_class_memory);
16510 %}
16511 
16512 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16513                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16514 %{
16515   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16516   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16517   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16518 
16519   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16520   ins_encode %{
16521     __ string_compare($str1$$Register, $str2$$Register,
16522                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16523                       $tmp1$$Register, $tmp2$$Register,
16524                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16525   %}
16526   ins_pipe(pipe_class_memory);
16527 %}
16528 
16529 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16530                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16531                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16532 %{
16533   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16534   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16535   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16536          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16537 
16538   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16539   ins_encode %{
16540     __ string_compare($str1$$Register, $str2$$Register,
16541                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16542                       $tmp1$$Register, $tmp2$$Register,
16543                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16544                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16545   %}
16546   ins_pipe(pipe_class_memory);
16547 %}
16548 
16549 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16550                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16551                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16552 %{
16553   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16554   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16555   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16556          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16557 
16558   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16559   ins_encode %{
16560     __ string_compare($str1$$Register, $str2$$Register,
16561                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16562                       $tmp1$$Register, $tmp2$$Register,
16563                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16564                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16565   %}
16566   ins_pipe(pipe_class_memory);
16567 %}
16568 
16569 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16570        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16571        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16572 %{
16573   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16574   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16575   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16576          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16577   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16578 
16579   ins_encode %{
16580     __ string_indexof($str1$$Register, $str2$$Register,
16581                       $cnt1$$Register, $cnt2$$Register,
16582                       $tmp1$$Register, $tmp2$$Register,
16583                       $tmp3$$Register, $tmp4$$Register,
16584                       $tmp5$$Register, $tmp6$$Register,
16585                       -1, $result$$Register, StrIntrinsicNode::UU);
16586   %}
16587   ins_pipe(pipe_class_memory);
16588 %}
16589 
16590 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16591        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16592        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16593 %{
16594   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16595   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16596   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16597          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16598   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16599 
16600   ins_encode %{
16601     __ string_indexof($str1$$Register, $str2$$Register,
16602                       $cnt1$$Register, $cnt2$$Register,
16603                       $tmp1$$Register, $tmp2$$Register,
16604                       $tmp3$$Register, $tmp4$$Register,
16605                       $tmp5$$Register, $tmp6$$Register,
16606                       -1, $result$$Register, StrIntrinsicNode::LL);
16607   %}
16608   ins_pipe(pipe_class_memory);
16609 %}
16610 
16611 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16612        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16613        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16614 %{
16615   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16616   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16617   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16618          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16619   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16620 
16621   ins_encode %{
16622     __ string_indexof($str1$$Register, $str2$$Register,
16623                       $cnt1$$Register, $cnt2$$Register,
16624                       $tmp1$$Register, $tmp2$$Register,
16625                       $tmp3$$Register, $tmp4$$Register,
16626                       $tmp5$$Register, $tmp6$$Register,
16627                       -1, $result$$Register, StrIntrinsicNode::UL);
16628   %}
16629   ins_pipe(pipe_class_memory);
16630 %}
16631 
16632 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16633                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16634                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16635 %{
16636   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16637   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16638   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16639          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16640   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16641 
16642   ins_encode %{
16643     int icnt2 = (int)$int_cnt2$$constant;
16644     __ string_indexof($str1$$Register, $str2$$Register,
16645                       $cnt1$$Register, zr,
16646                       $tmp1$$Register, $tmp2$$Register,
16647                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16648                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16649   %}
16650   ins_pipe(pipe_class_memory);
16651 %}
16652 
16653 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16654                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16655                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16656 %{
16657   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16658   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16659   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16660          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16661   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16662 
16663   ins_encode %{
16664     int icnt2 = (int)$int_cnt2$$constant;
16665     __ string_indexof($str1$$Register, $str2$$Register,
16666                       $cnt1$$Register, zr,
16667                       $tmp1$$Register, $tmp2$$Register,
16668                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16669                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16670   %}
16671   ins_pipe(pipe_class_memory);
16672 %}
16673 
16674 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16675                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16676                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16677 %{
16678   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16679   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16680   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16681          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16682   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16683 
16684   ins_encode %{
16685     int icnt2 = (int)$int_cnt2$$constant;
16686     __ string_indexof($str1$$Register, $str2$$Register,
16687                       $cnt1$$Register, zr,
16688                       $tmp1$$Register, $tmp2$$Register,
16689                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16690                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16691   %}
16692   ins_pipe(pipe_class_memory);
16693 %}
16694 
16695 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16696                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16697                              iRegINoSp tmp3, rFlagsReg cr)
16698 %{
16699   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16700   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16701   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16702          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16703 
16704   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16705 
16706   ins_encode %{
16707     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16708                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16709                            $tmp3$$Register);
16710   %}
16711   ins_pipe(pipe_class_memory);
16712 %}
16713 
16714 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16715                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16716                               iRegINoSp tmp3, rFlagsReg cr)
16717 %{
16718   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16719   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16720   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16721          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16722 
16723   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16724 
16725   ins_encode %{
16726     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16727                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16728                             $tmp3$$Register);
16729   %}
16730   ins_pipe(pipe_class_memory);
16731 %}
16732 
16733 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16734                         iRegI_R0 result, rFlagsReg cr)
16735 %{
16736   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16737   match(Set result (StrEquals (Binary str1 str2) cnt));
16738   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16739 
16740   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16741   ins_encode %{
16742     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16743     __ string_equals($str1$$Register, $str2$$Register,
16744                      $result$$Register, $cnt$$Register, 1);
16745   %}
16746   ins_pipe(pipe_class_memory);
16747 %}
16748 
16749 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16750                         iRegI_R0 result, rFlagsReg cr)
16751 %{
16752   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16753   match(Set result (StrEquals (Binary str1 str2) cnt));
16754   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16755 
16756   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16757   ins_encode %{
16758     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16759     __ string_equals($str1$$Register, $str2$$Register,
16760                      $result$$Register, $cnt$$Register, 2);
16761   %}
16762   ins_pipe(pipe_class_memory);
16763 %}
16764 
16765 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16766                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16767                        iRegP_R10 tmp, rFlagsReg cr)
16768 %{
16769   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16770   match(Set result (AryEq ary1 ary2));
16771   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16772 
16773   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16774   ins_encode %{
16775     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16776                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16777                                    $result$$Register, $tmp$$Register, 1);
16778     if (tpc == NULL) {
16779       ciEnv::current()->record_failure("CodeCache is full");
16780       return;
16781     }
16782   %}
16783   ins_pipe(pipe_class_memory);
16784 %}
16785 
16786 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16787                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16788                        iRegP_R10 tmp, rFlagsReg cr)
16789 %{
16790   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16791   match(Set result (AryEq ary1 ary2));
16792   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16793 
16794   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16795   ins_encode %{
16796     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16797                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16798                                    $result$$Register, $tmp$$Register, 2);
16799     if (tpc == NULL) {
16800       ciEnv::current()->record_failure("CodeCache is full");
16801       return;
16802     }
16803   %}
16804   ins_pipe(pipe_class_memory);
16805 %}
16806 
16807 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16808 %{
16809   match(Set result (HasNegatives ary1 len));
16810   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16811   format %{ "has negatives byte[] $ary1,$len -> $result" %}
16812   ins_encode %{
16813     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
16814     if (tpc == NULL) {
16815       ciEnv::current()->record_failure("CodeCache is full");
16816       return;
16817     }
16818   %}
16819   ins_pipe( pipe_slow );
16820 %}
16821 
16822 // fast char[] to byte[] compression
16823 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16824                          vRegD_V0 tmp1, vRegD_V1 tmp2,
16825                          vRegD_V2 tmp3, vRegD_V3 tmp4,
16826                          iRegI_R0 result, rFlagsReg cr)
16827 %{
16828   match(Set result (StrCompressedCopy src (Binary dst len)));
16829   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16830 
16831   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
16832   ins_encode %{
16833     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16834                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16835                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
16836                            $result$$Register);
16837   %}
16838   ins_pipe( pipe_slow );
16839 %}
16840 
16841 // fast byte[] to char[] inflation
16842 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16843                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16844 %{
16845   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16846   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16847 
16848   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16849   ins_encode %{
16850     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16851                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16852                                         $tmp3$$FloatRegister, $tmp4$$Register);
16853     if (tpc == NULL) {
16854       ciEnv::current()->record_failure("CodeCache is full");
16855       return;
16856     }
16857   %}
16858   ins_pipe(pipe_class_memory);
16859 %}
16860 
16861 // encode char[] to byte[] in ISO_8859_1
16862 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16863                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
16864                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
16865                           iRegI_R0 result, rFlagsReg cr)
16866 %{
16867   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16868   match(Set result (EncodeISOArray src (Binary dst len)));
16869   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
16870          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
16871 
16872   format %{ "Encode array $src,$dst,$len -> $result" %}
16873   ins_encode %{
16874     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16875          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
16876          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
16877   %}
16878   ins_pipe( pipe_class_memory );
16879 %}
16880 
16881 // ============================================================================
16882 // This name is KNOWN by the ADLC and cannot be changed.
16883 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16884 // for this guy.
16885 instruct tlsLoadP(thread_RegP dst)
16886 %{
16887   match(Set dst (ThreadLocal));
16888 
16889   ins_cost(0);
16890 
16891   format %{ " -- \t// $dst=Thread::current(), empty" %}
16892 
16893   size(0);
16894 
16895   ins_encode( /*empty*/ );
16896 
16897   ins_pipe(pipe_class_empty);
16898 %}
16899 
16900 //----------PEEPHOLE RULES-----------------------------------------------------
16901 // These must follow all instruction definitions as they use the names
16902 // defined in the instructions definitions.
16903 //
16904 // peepmatch ( root_instr_name [preceding_instruction]* );
16905 //
16906 // peepconstraint %{
16907 // (instruction_number.operand_name relational_op instruction_number.operand_name
16908 //  [, ...] );
16909 // // instruction numbers are zero-based using left to right order in peepmatch
16910 //
16911 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
16912 // // provide an instruction_number.operand_name for each operand that appears
16913 // // in the replacement instruction's match rule
16914 //
16915 // ---------VM FLAGS---------------------------------------------------------
16916 //
16917 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16918 //
16919 // Each peephole rule is given an identifying number starting with zero and
16920 // increasing by one in the order seen by the parser.  An individual peephole
16921 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16922 // on the command-line.
16923 //
16924 // ---------CURRENT LIMITATIONS----------------------------------------------
16925 //
16926 // Only match adjacent instructions in same basic block
16927 // Only equality constraints
16928 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16929 // Only one replacement instruction
16930 //
16931 // ---------EXAMPLE----------------------------------------------------------
16932 //
16933 // // pertinent parts of existing instructions in architecture description
16934 // instruct movI(iRegINoSp dst, iRegI src)
16935 // %{
16936 //   match(Set dst (CopyI src));
16937 // %}
16938 //
16939 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16940 // %{
16941 //   match(Set dst (AddI dst src));
16942 //   effect(KILL cr);
16943 // %}
16944 //
16945 // // Change (inc mov) to lea
16946 // peephole %{
16947 //   // increment preceeded by register-register move
16948 //   peepmatch ( incI_iReg movI );
16949 //   // require that the destination register of the increment
16950 //   // match the destination register of the move
16951 //   peepconstraint ( 0.dst == 1.dst );
16952 //   // construct a replacement instruction that sets
16953 //   // the destination to ( move's source register + one )
16954 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16955 // %}
16956 //
16957 
16958 // Implementation no longer uses movX instructions since
16959 // machine-independent system no longer uses CopyX nodes.
16960 //
16961 // peephole
16962 // %{
16963 //   peepmatch (incI_iReg movI);
16964 //   peepconstraint (0.dst == 1.dst);
16965 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16966 // %}
16967 
16968 // peephole
16969 // %{
16970 //   peepmatch (decI_iReg movI);
16971 //   peepconstraint (0.dst == 1.dst);
16972 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16973 // %}
16974 
16975 // peephole
16976 // %{
16977 //   peepmatch (addI_iReg_imm movI);
16978 //   peepconstraint (0.dst == 1.dst);
16979 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16980 // %}
16981 
16982 // peephole
16983 // %{
16984 //   peepmatch (incL_iReg movL);
16985 //   peepconstraint (0.dst == 1.dst);
16986 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16987 // %}
16988 
16989 // peephole
16990 // %{
16991 //   peepmatch (decL_iReg movL);
16992 //   peepconstraint (0.dst == 1.dst);
16993 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16994 // %}
16995 
16996 // peephole
16997 // %{
16998 //   peepmatch (addL_iReg_imm movL);
16999 //   peepconstraint (0.dst == 1.dst);
17000 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17001 // %}
17002 
17003 // peephole
17004 // %{
17005 //   peepmatch (addP_iReg_imm movP);
17006 //   peepconstraint (0.dst == 1.dst);
17007 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17008 // %}
17009 
17010 // // Change load of spilled value to only a spill
17011 // instruct storeI(memory mem, iRegI src)
17012 // %{
17013 //   match(Set mem (StoreI mem src));
17014 // %}
17015 //
17016 // instruct loadI(iRegINoSp dst, memory mem)
17017 // %{
17018 //   match(Set dst (LoadI mem));
17019 // %}
17020 //
17021 
17022 //----------SMARTSPILL RULES---------------------------------------------------
17023 // These must follow all instruction definitions as they use the names
17024 // defined in the instructions definitions.
17025 
17026 // Local Variables:
17027 // mode: c++
17028 // End: