1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return 4 * NativeInstruction::instruction_size;
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298   bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // Assert that the given node is not a variable shift.
 1315   bool assert_not_var_shift(const Node* n);
 1316 
 1317   // predicate controlling addressing modes
 1318   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1319 %}
 1320 
 1321 source %{
 1322 
 1323   // Derived RegMask with conditionally allocatable registers
 1324 
 1325   void PhaseOutput::pd_perform_mach_node_analysis() {
 1326   }
 1327 
 1328   int MachNode::pd_alignment_required() const {
 1329     return 1;
 1330   }
 1331 
 1332   int MachNode::compute_padding(int current_offset) const {
 1333     return 0;
 1334   }
 1335 
 1336   RegMask _ANY_REG32_mask;
 1337   RegMask _ANY_REG_mask;
 1338   RegMask _PTR_REG_mask;
 1339   RegMask _NO_SPECIAL_REG32_mask;
 1340   RegMask _NO_SPECIAL_REG_mask;
 1341   RegMask _NO_SPECIAL_PTR_REG_mask;
 1342 
 1343   void reg_mask_init() {
 1344     // We derive below RegMask(s) from the ones which are auto-generated from
 1345     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1346     // registers conditionally reserved.
 1347 
 1348     _ANY_REG32_mask = _ALL_REG32_mask;
 1349     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1350 
 1351     _ANY_REG_mask = _ALL_REG_mask;
 1352 
 1353     _PTR_REG_mask = _ALL_REG_mask;
 1354 
 1355     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1356     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1357 
 1358     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1362     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1363 
 1364     // r27 is not allocatable when compressed oops is on and heapbase is not
 1365     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1366     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1367       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1368       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1369       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1370     }
 1371 
 1372     // r29 is not allocatable when PreserveFramePointer is on
 1373     if (PreserveFramePointer) {
 1374       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1375       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1376       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1377     }
 1378   }
 1379 
 1380   // Optimizaton of volatile gets and puts
 1381   // -------------------------------------
 1382   //
 1383   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1384   // use to implement volatile reads and writes. For a volatile read
 1385   // we simply need
 1386   //
 1387   //   ldar<x>
 1388   //
 1389   // and for a volatile write we need
 1390   //
 1391   //   stlr<x>
 1392   //
 1393   // Alternatively, we can implement them by pairing a normal
 1394   // load/store with a memory barrier. For a volatile read we need
 1395   //
 1396   //   ldr<x>
 1397   //   dmb ishld
 1398   //
 1399   // for a volatile write
 1400   //
 1401   //   dmb ish
 1402   //   str<x>
 1403   //   dmb ish
 1404   //
 1405   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1406   // sequences. These are normally translated to an instruction
 1407   // sequence like the following
 1408   //
 1409   //   dmb      ish
 1410   // retry:
 1411   //   ldxr<x>   rval raddr
 1412   //   cmp       rval rold
 1413   //   b.ne done
 1414   //   stlxr<x>  rval, rnew, rold
 1415   //   cbnz      rval retry
 1416   // done:
 1417   //   cset      r0, eq
 1418   //   dmb ishld
 1419   //
 1420   // Note that the exclusive store is already using an stlxr
 1421   // instruction. That is required to ensure visibility to other
 1422   // threads of the exclusive write (assuming it succeeds) before that
 1423   // of any subsequent writes.
 1424   //
 1425   // The following instruction sequence is an improvement on the above
 1426   //
 1427   // retry:
 1428   //   ldaxr<x>  rval raddr
 1429   //   cmp       rval rold
 1430   //   b.ne done
 1431   //   stlxr<x>  rval, rnew, rold
 1432   //   cbnz      rval retry
 1433   // done:
 1434   //   cset      r0, eq
 1435   //
 1436   // We don't need the leading dmb ish since the stlxr guarantees
 1437   // visibility of prior writes in the case that the swap is
 1438   // successful. Crucially we don't have to worry about the case where
 1439   // the swap is not successful since no valid program should be
 1440   // relying on visibility of prior changes by the attempting thread
 1441   // in the case where the CAS fails.
 1442   //
 1443   // Similarly, we don't need the trailing dmb ishld if we substitute
 1444   // an ldaxr instruction since that will provide all the guarantees we
 1445   // require regarding observation of changes made by other threads
 1446   // before any change to the CAS address observed by the load.
 1447   //
 1448   // In order to generate the desired instruction sequence we need to
 1449   // be able to identify specific 'signature' ideal graph node
 1450   // sequences which i) occur as a translation of a volatile reads or
 1451   // writes or CAS operations and ii) do not occur through any other
 1452   // translation or graph transformation. We can then provide
 1453   // alternative aldc matching rules which translate these node
 1454   // sequences to the desired machine code sequences. Selection of the
 1455   // alternative rules can be implemented by predicates which identify
 1456   // the relevant node sequences.
 1457   //
 1458   // The ideal graph generator translates a volatile read to the node
 1459   // sequence
 1460   //
 1461   //   LoadX[mo_acquire]
 1462   //   MemBarAcquire
 1463   //
 1464   // As a special case when using the compressed oops optimization we
 1465   // may also see this variant
 1466   //
 1467   //   LoadN[mo_acquire]
 1468   //   DecodeN
 1469   //   MemBarAcquire
 1470   //
 1471   // A volatile write is translated to the node sequence
 1472   //
 1473   //   MemBarRelease
 1474   //   StoreX[mo_release] {CardMark}-optional
 1475   //   MemBarVolatile
 1476   //
 1477   // n.b. the above node patterns are generated with a strict
 1478   // 'signature' configuration of input and output dependencies (see
 1479   // the predicates below for exact details). The card mark may be as
 1480   // simple as a few extra nodes or, in a few GC configurations, may
 1481   // include more complex control flow between the leading and
 1482   // trailing memory barriers. However, whatever the card mark
 1483   // configuration these signatures are unique to translated volatile
 1484   // reads/stores -- they will not appear as a result of any other
 1485   // bytecode translation or inlining nor as a consequence of
 1486   // optimizing transforms.
 1487   //
 1488   // We also want to catch inlined unsafe volatile gets and puts and
 1489   // be able to implement them using either ldar<x>/stlr<x> or some
 1490   // combination of ldr<x>/stlr<x> and dmb instructions.
 1491   //
 1492   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1493   // normal volatile put node sequence containing an extra cpuorder
 1494   // membar
 1495   //
 1496   //   MemBarRelease
 1497   //   MemBarCPUOrder
 1498   //   StoreX[mo_release] {CardMark}-optional
 1499   //   MemBarCPUOrder
 1500   //   MemBarVolatile
 1501   //
 1502   // n.b. as an aside, a cpuorder membar is not itself subject to
 1503   // matching and translation by adlc rules.  However, the rule
 1504   // predicates need to detect its presence in order to correctly
 1505   // select the desired adlc rules.
 1506   //
 1507   // Inlined unsafe volatile gets manifest as a slightly different
 1508   // node sequence to a normal volatile get because of the
 1509   // introduction of some CPUOrder memory barriers to bracket the
 1510   // Load. However, but the same basic skeleton of a LoadX feeding a
 1511   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1512   // present
 1513   //
 1514   //   MemBarCPUOrder
 1515   //        ||       \\
 1516   //   MemBarCPUOrder LoadX[mo_acquire]
 1517   //        ||            |
 1518   //        ||       {DecodeN} optional
 1519   //        ||       /
 1520   //     MemBarAcquire
 1521   //
 1522   // In this case the acquire membar does not directly depend on the
 1523   // load. However, we can be sure that the load is generated from an
 1524   // inlined unsafe volatile get if we see it dependent on this unique
 1525   // sequence of membar nodes. Similarly, given an acquire membar we
 1526   // can know that it was added because of an inlined unsafe volatile
 1527   // get if it is fed and feeds a cpuorder membar and if its feed
 1528   // membar also feeds an acquiring load.
 1529   //
 1530   // Finally an inlined (Unsafe) CAS operation is translated to the
 1531   // following ideal graph
 1532   //
 1533   //   MemBarRelease
 1534   //   MemBarCPUOrder
 1535   //   CompareAndSwapX {CardMark}-optional
 1536   //   MemBarCPUOrder
 1537   //   MemBarAcquire
 1538   //
 1539   // So, where we can identify these volatile read and write
 1540   // signatures we can choose to plant either of the above two code
 1541   // sequences. For a volatile read we can simply plant a normal
 1542   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1543   // also choose to inhibit translation of the MemBarAcquire and
 1544   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1545   //
 1546   // When we recognise a volatile store signature we can choose to
 1547   // plant at a dmb ish as a translation for the MemBarRelease, a
 1548   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1549   // Alternatively, we can inhibit translation of the MemBarRelease
 1550   // and MemBarVolatile and instead plant a simple stlr<x>
 1551   // instruction.
 1552   //
 1553   // when we recognise a CAS signature we can choose to plant a dmb
 1554   // ish as a translation for the MemBarRelease, the conventional
 1555   // macro-instruction sequence for the CompareAndSwap node (which
 1556   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1557   // Alternatively, we can elide generation of the dmb instructions
 1558   // and plant the alternative CompareAndSwap macro-instruction
 1559   // sequence (which uses ldaxr<x>).
 1560   //
 1561   // Of course, the above only applies when we see these signature
 1562   // configurations. We still want to plant dmb instructions in any
 1563   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1564   // MemBarVolatile. For example, at the end of a constructor which
 1565   // writes final/volatile fields we will see a MemBarRelease
 1566   // instruction and this needs a 'dmb ish' lest we risk the
 1567   // constructed object being visible without making the
 1568   // final/volatile field writes visible.
 1569   //
 1570   // n.b. the translation rules below which rely on detection of the
 1571   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1572   // If we see anything other than the signature configurations we
 1573   // always just translate the loads and stores to ldr<x> and str<x>
 1574   // and translate acquire, release and volatile membars to the
 1575   // relevant dmb instructions.
 1576   //
 1577 
 1578   // is_CAS(int opcode, bool maybe_volatile)
 1579   //
 1580   // return true if opcode is one of the possible CompareAndSwapX
 1581   // values otherwise false.
 1582 
 1583   bool is_CAS(int opcode, bool maybe_volatile)
 1584   {
 1585     switch(opcode) {
 1586       // We handle these
 1587     case Op_CompareAndSwapI:
 1588     case Op_CompareAndSwapL:
 1589     case Op_CompareAndSwapP:
 1590     case Op_CompareAndSwapN:
 1591     case Op_ShenandoahCompareAndSwapP:
 1592     case Op_ShenandoahCompareAndSwapN:
 1593     case Op_CompareAndSwapB:
 1594     case Op_CompareAndSwapS:
 1595     case Op_GetAndSetI:
 1596     case Op_GetAndSetL:
 1597     case Op_GetAndSetP:
 1598     case Op_GetAndSetN:
 1599     case Op_GetAndAddI:
 1600     case Op_GetAndAddL:
 1601       return true;
 1602     case Op_CompareAndExchangeI:
 1603     case Op_CompareAndExchangeN:
 1604     case Op_CompareAndExchangeB:
 1605     case Op_CompareAndExchangeS:
 1606     case Op_CompareAndExchangeL:
 1607     case Op_CompareAndExchangeP:
 1608     case Op_WeakCompareAndSwapB:
 1609     case Op_WeakCompareAndSwapS:
 1610     case Op_WeakCompareAndSwapI:
 1611     case Op_WeakCompareAndSwapL:
 1612     case Op_WeakCompareAndSwapP:
 1613     case Op_WeakCompareAndSwapN:
 1614     case Op_ShenandoahWeakCompareAndSwapP:
 1615     case Op_ShenandoahWeakCompareAndSwapN:
 1616     case Op_ShenandoahCompareAndExchangeP:
 1617     case Op_ShenandoahCompareAndExchangeN:
 1618       return maybe_volatile;
 1619     default:
 1620       return false;
 1621     }
 1622   }
 1623 
 1624   // helper to determine the maximum number of Phi nodes we may need to
 1625   // traverse when searching from a card mark membar for the merge mem
 1626   // feeding a trailing membar or vice versa
 1627 
 1628 // predicates controlling emit of ldr<x>/ldar<x>
 1629 
 1630 bool unnecessary_acquire(const Node *barrier)
 1631 {
 1632   assert(barrier->is_MemBar(), "expecting a membar");
 1633 
 1634   MemBarNode* mb = barrier->as_MemBar();
 1635 
 1636   if (mb->trailing_load()) {
 1637     return true;
 1638   }
 1639 
 1640   if (mb->trailing_load_store()) {
 1641     Node* load_store = mb->in(MemBarNode::Precedent);
 1642     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1643     return is_CAS(load_store->Opcode(), true);
 1644   }
 1645 
 1646   return false;
 1647 }
 1648 
 1649 bool needs_acquiring_load(const Node *n)
 1650 {
 1651   assert(n->is_Load(), "expecting a load");
 1652   LoadNode *ld = n->as_Load();
 1653   return ld->is_acquire();
 1654 }
 1655 
 1656 bool unnecessary_release(const Node *n)
 1657 {
 1658   assert((n->is_MemBar() &&
 1659           n->Opcode() == Op_MemBarRelease),
 1660          "expecting a release membar");
 1661 
 1662   MemBarNode *barrier = n->as_MemBar();
 1663   if (!barrier->leading()) {
 1664     return false;
 1665   } else {
 1666     Node* trailing = barrier->trailing_membar();
 1667     MemBarNode* trailing_mb = trailing->as_MemBar();
 1668     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1669     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1670 
 1671     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1672     if (mem->is_Store()) {
 1673       assert(mem->as_Store()->is_release(), "");
 1674       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1675       return true;
 1676     } else {
 1677       assert(mem->is_LoadStore(), "");
 1678       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1679       return is_CAS(mem->Opcode(), true);
 1680     }
 1681   }
 1682   return false;
 1683 }
 1684 
 1685 bool unnecessary_volatile(const Node *n)
 1686 {
 1687   // assert n->is_MemBar();
 1688   MemBarNode *mbvol = n->as_MemBar();
 1689 
 1690   bool release = mbvol->trailing_store();
 1691   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1692 #ifdef ASSERT
 1693   if (release) {
 1694     Node* leading = mbvol->leading_membar();
 1695     assert(leading->Opcode() == Op_MemBarRelease, "");
 1696     assert(leading->as_MemBar()->leading_store(), "");
 1697     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1698   }
 1699 #endif
 1700 
 1701   return release;
 1702 }
 1703 
 1704 // predicates controlling emit of str<x>/stlr<x>
 1705 
 1706 bool needs_releasing_store(const Node *n)
 1707 {
 1708   // assert n->is_Store();
 1709   StoreNode *st = n->as_Store();
 1710   return st->trailing_membar() != NULL;
 1711 }
 1712 
 1713 // predicate controlling translation of CAS
 1714 //
 1715 // returns true if CAS needs to use an acquiring load otherwise false
 1716 
 1717 bool needs_acquiring_load_exclusive(const Node *n)
 1718 {
 1719   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1720   LoadStoreNode* ldst = n->as_LoadStore();
 1721   if (is_CAS(n->Opcode(), false)) {
 1722     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1723   } else {
 1724     return ldst->trailing_membar() != NULL;
 1725   }
 1726 
 1727   // so we can just return true here
 1728   return true;
 1729 }
 1730 
 1731 // Assert that the given node is not a variable shift.
 1732 bool assert_not_var_shift(const Node* n) {
 1733   assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
 1734   return true;
 1735 }
 1736 
 1737 #define __ _masm.
 1738 
 1739 // advance declarations for helper functions to convert register
 1740 // indices to register objects
 1741 
 1742 // the ad file has to provide implementations of certain methods
 1743 // expected by the generic code
 1744 //
 1745 // REQUIRED FUNCTIONALITY
 1746 
 1747 //=============================================================================
 1748 
 1749 // !!!!! Special hack to get all types of calls to specify the byte offset
 1750 //       from the start of the call to the point where the return address
 1751 //       will point.
 1752 
 1753 int MachCallStaticJavaNode::ret_addr_offset()
 1754 {
 1755   // call should be a simple bl
 1756   int off = 4;
 1757   return off;
 1758 }
 1759 
 1760 int MachCallDynamicJavaNode::ret_addr_offset()
 1761 {
 1762   return 16; // movz, movk, movk, bl
 1763 }
 1764 
 1765 int MachCallRuntimeNode::ret_addr_offset() {
 1766   // for generated stubs the call will be
 1767   //   bl(addr)
 1768   // or with far branches
 1769   //   bl(trampoline_stub)
 1770   // for real runtime callouts it will be six instructions
 1771   // see aarch64_enc_java_to_runtime
 1772   //   adr(rscratch2, retaddr)
 1773   //   lea(rscratch1, RuntimeAddress(addr)
 1774   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1775   //   blr(rscratch1)
 1776   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1777   if (cb) {
 1778     return 1 * NativeInstruction::instruction_size;
 1779   } else if (_entry_point == NULL) {
 1780     // See CallLeafNoFPIndirect
 1781     return 1 * NativeInstruction::instruction_size;
 1782   } else {
 1783     return 6 * NativeInstruction::instruction_size;
 1784   }
 1785 }
 1786 
 1787 int MachCallNativeNode::ret_addr_offset() {
 1788   // This is implemented using aarch64_enc_java_to_runtime as above.
 1789   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1790   if (cb) {
 1791     return 1 * NativeInstruction::instruction_size;
 1792   } else {
 1793     return 6 * NativeInstruction::instruction_size;
 1794   }
 1795 }
 1796 
 1797 //=============================================================================
 1798 
 1799 #ifndef PRODUCT
 1800 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1801   st->print("BREAKPOINT");
 1802 }
 1803 #endif
 1804 
 1805 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1806   C2_MacroAssembler _masm(&cbuf);
 1807   __ brk(0);
 1808 }
 1809 
 1810 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1811   return MachNode::size(ra_);
 1812 }
 1813 
 1814 //=============================================================================
 1815 
 1816 #ifndef PRODUCT
 1817   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1818     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1819   }
 1820 #endif
 1821 
 1822   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1823     C2_MacroAssembler _masm(&cbuf);
 1824     for (int i = 0; i < _count; i++) {
 1825       __ nop();
 1826     }
 1827   }
 1828 
 1829   uint MachNopNode::size(PhaseRegAlloc*) const {
 1830     return _count * NativeInstruction::instruction_size;
 1831   }
 1832 
 1833 //=============================================================================
 1834 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1835 
 1836 int ConstantTable::calculate_table_base_offset() const {
 1837   return 0;  // absolute addressing, no offset
 1838 }
 1839 
 1840 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1841 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1842   ShouldNotReachHere();
 1843 }
 1844 
 1845 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1846   // Empty encoding
 1847 }
 1848 
 1849 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1850   return 0;
 1851 }
 1852 
 1853 #ifndef PRODUCT
 1854 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1855   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1856 }
 1857 #endif
 1858 
 1859 #ifndef PRODUCT
 1860 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1861   Compile* C = ra_->C;
 1862 
 1863   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1864 
 1865   if (C->output()->need_stack_bang(framesize))
 1866     st->print("# stack bang size=%d\n\t", framesize);
 1867 
 1868   if (VM_Version::use_rop_protection()) {
 1869     st->print("ldr zr, [lr]\n\t");
 1870     st->print("pacia  lr, rfp\n\t");
 1871   }
 1872   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1873     st->print("sub  sp, sp, #%d\n\t", framesize);
 1874     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1875     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1876   } else {
 1877     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1878     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1879     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1880     st->print("sub  sp, sp, rscratch1");
 1881   }
 1882   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1883     st->print("\n\t");
 1884     st->print("ldr  rscratch1, [guard]\n\t");
 1885     st->print("dmb ishld\n\t");
 1886     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1887     st->print("cmp  rscratch1, rscratch2\n\t");
 1888     st->print("b.eq skip");
 1889     st->print("\n\t");
 1890     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1891     st->print("b skip\n\t");
 1892     st->print("guard: int\n\t");
 1893     st->print("\n\t");
 1894     st->print("skip:\n\t");
 1895   }
 1896 }
 1897 #endif
 1898 
 1899 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1900   Compile* C = ra_->C;
 1901   C2_MacroAssembler _masm(&cbuf);
 1902 
 1903   // insert a nop at the start of the prolog so we can patch in a
 1904   // branch if we need to invalidate the method later
 1905   __ nop();
 1906 
 1907   if (C->clinit_barrier_on_entry()) {
 1908     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1909 
 1910     Label L_skip_barrier;
 1911 
 1912     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1913     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1914     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1915     __ bind(L_skip_barrier);
 1916   }
 1917 
 1918   if (C->max_vector_size() > 0) {
 1919     __ reinitialize_ptrue();
 1920   }
 1921 
 1922   __ verified_entry(C, 0);
 1923   __ bind(*_verified_entry);
 1924 
 1925   if (C->stub_function() == NULL) {
 1926     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1927     bs->nmethod_entry_barrier(&_masm);
 1928   }
 1929 
 1930   if (VerifyStackAtCalls) {
 1931     Unimplemented();
 1932   }
 1933 
 1934   C->output()->set_frame_complete(cbuf.insts_size());
 1935 
 1936   if (C->has_mach_constant_base_node()) {
 1937     // NOTE: We set the table base offset here because users might be
 1938     // emitted before MachConstantBaseNode.
 1939     ConstantTable& constant_table = C->output()->constant_table();
 1940     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1941   }
 1942 }
 1943 
 1944 int MachPrologNode::reloc() const
 1945 {
 1946   return 0;
 1947 }
 1948 
 1949 //=============================================================================
 1950 
 1951 #ifndef PRODUCT
 1952 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1953   Compile* C = ra_->C;
 1954   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1955 
 1956   st->print("# pop frame %d\n\t",framesize);
 1957 
 1958   if (framesize == 0) {
 1959     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1960   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1961     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1962     st->print("add  sp, sp, #%d\n\t", framesize);
 1963   } else {
 1964     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1965     st->print("add  sp, sp, rscratch1\n\t");
 1966     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1967   }
 1968   if (VM_Version::use_rop_protection()) {
 1969     st->print("autia lr, rfp\n\t");
 1970     st->print("ldr zr, [lr]\n\t");
 1971   }
 1972 
 1973   if (do_polling() && C->is_method_compilation()) {
 1974     st->print("# test polling word\n\t");
 1975     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1976     st->print("cmp  sp, rscratch1\n\t");
 1977     st->print("bhi #slow_path");
 1978   }
 1979 }
 1980 #endif
 1981 
 1982 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1983   Compile* C = ra_->C;
 1984   C2_MacroAssembler _masm(&cbuf);
 1985   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1986 
 1987   __ remove_frame(framesize, C->needs_stack_repair());
 1988 
 1989   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1990     __ reserved_stack_check();
 1991   }
 1992 
 1993   if (do_polling() && C->is_method_compilation()) {
 1994     Label dummy_label;
 1995     Label* code_stub = &dummy_label;
 1996     if (!C->output()->in_scratch_emit_size()) {
 1997       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1998     }
 1999     __ relocate(relocInfo::poll_return_type);
 2000     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 2001   }
 2002 }
 2003 
 2004 int MachEpilogNode::reloc() const {
 2005   // Return number of relocatable values contained in this instruction.
 2006   return 1; // 1 for polling page.
 2007 }
 2008 
 2009 const Pipeline * MachEpilogNode::pipeline() const {
 2010   return MachNode::pipeline_class();
 2011 }
 2012 
 2013 //=============================================================================
 2014 
 2015 // Figure out which register class each belongs in: rc_int, rc_float or
 2016 // rc_stack.
 2017 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2018 
 2019 static enum RC rc_class(OptoReg::Name reg) {
 2020 
 2021   if (reg == OptoReg::Bad) {
 2022     return rc_bad;
 2023   }
 2024 
 2025   // we have 32 int registers * 2 halves
 2026   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2027 
 2028   if (reg < slots_of_int_registers) {
 2029     return rc_int;
 2030   }
 2031 
 2032   // we have 32 float register * 8 halves
 2033   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2034   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2035     return rc_float;
 2036   }
 2037 
 2038   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2039   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2040     return rc_predicate;
 2041   }
 2042 
 2043   // Between predicate regs & stack is the flags.
 2044   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2045 
 2046   return rc_stack;
 2047 }
 2048 
 2049 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2050   Compile* C = ra_->C;
 2051 
 2052   // Get registers to move.
 2053   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2054   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2055   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2056   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2057 
 2058   enum RC src_hi_rc = rc_class(src_hi);
 2059   enum RC src_lo_rc = rc_class(src_lo);
 2060   enum RC dst_hi_rc = rc_class(dst_hi);
 2061   enum RC dst_lo_rc = rc_class(dst_lo);
 2062 
 2063   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2064 
 2065   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 2066     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2067            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2068            "expected aligned-adjacent pairs");
 2069   }
 2070 
 2071   if (src_lo == dst_lo && src_hi == dst_hi) {
 2072     return 0;            // Self copy, no move.
 2073   }
 2074 
 2075   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2076               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2077   int src_offset = ra_->reg2offset(src_lo);
 2078   int dst_offset = ra_->reg2offset(dst_lo);
 2079 
 2080   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2081     uint ireg = ideal_reg();
 2082     if (ireg == Op_VecA && cbuf) {
 2083       C2_MacroAssembler _masm(cbuf);
 2084       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2085       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2086         // stack->stack
 2087         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2088                                                 sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2090         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2091                             sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2093         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2094                               sve_vector_reg_size_in_bytes);
 2095       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2096         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2097                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2098                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2099       } else {
 2100         ShouldNotReachHere();
 2101       }
 2102     } else if (cbuf) {
 2103       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2104       C2_MacroAssembler _masm(cbuf);
 2105       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2106       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2107         // stack->stack
 2108         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2109         if (ireg == Op_VecD) {
 2110           __ unspill(rscratch1, true, src_offset);
 2111           __ spill(rscratch1, true, dst_offset);
 2112         } else {
 2113           __ spill_copy128(src_offset, dst_offset);
 2114         }
 2115       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2116         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2117                ireg == Op_VecD ? __ T8B : __ T16B,
 2118                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2119       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2120         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2121                  ireg == Op_VecD ? __ D : __ Q,
 2122                  ra_->reg2offset(dst_lo));
 2123       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2124         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2125                    ireg == Op_VecD ? __ D : __ Q,
 2126                    ra_->reg2offset(src_lo));
 2127       } else {
 2128         ShouldNotReachHere();
 2129       }
 2130     }
 2131   } else if (cbuf) {
 2132     C2_MacroAssembler _masm(cbuf);
 2133     switch (src_lo_rc) {
 2134     case rc_int:
 2135       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2136         if (is64) {
 2137             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2138                    as_Register(Matcher::_regEncode[src_lo]));
 2139         } else {
 2140             C2_MacroAssembler _masm(cbuf);
 2141             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2142                     as_Register(Matcher::_regEncode[src_lo]));
 2143         }
 2144       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2145         if (is64) {
 2146             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2147                      as_Register(Matcher::_regEncode[src_lo]));
 2148         } else {
 2149             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2150                      as_Register(Matcher::_regEncode[src_lo]));
 2151         }
 2152       } else {                    // gpr --> stack spill
 2153         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2154         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2155       }
 2156       break;
 2157     case rc_float:
 2158       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2159         if (is64) {
 2160             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2161                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2162         } else {
 2163             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2164                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2165         }
 2166       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2167         if (is64) {
 2168             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         } else {
 2171             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2172                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2173         }
 2174       } else {                    // fpr --> stack spill
 2175         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2176         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2177                  is64 ? __ D : __ S, dst_offset);
 2178       }
 2179       break;
 2180     case rc_stack:
 2181       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2182         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2183       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2184         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2185                    is64 ? __ D : __ S, src_offset);
 2186       } else if (dst_lo_rc == rc_predicate) {
 2187         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2188                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2189       } else {                    // stack --> stack copy
 2190         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2191         if (ideal_reg() == Op_RegVectMask) {
 2192           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2193                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2194         } else {
 2195           __ unspill(rscratch1, is64, src_offset);
 2196           __ spill(rscratch1, is64, dst_offset);
 2197         }
 2198       }
 2199       break;
 2200     case rc_predicate:
 2201       if (dst_lo_rc == rc_predicate) {
 2202         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2203       } else if (dst_lo_rc == rc_stack) {
 2204         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2205                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2206       } else {
 2207         assert(false, "bad src and dst rc_class combination.");
 2208         ShouldNotReachHere();
 2209       }
 2210       break;
 2211     default:
 2212       assert(false, "bad rc_class for spill");
 2213       ShouldNotReachHere();
 2214     }
 2215   }
 2216 
 2217   if (st) {
 2218     st->print("spill ");
 2219     if (src_lo_rc == rc_stack) {
 2220       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2221     } else {
 2222       st->print("%s -> ", Matcher::regName[src_lo]);
 2223     }
 2224     if (dst_lo_rc == rc_stack) {
 2225       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2226     } else {
 2227       st->print("%s", Matcher::regName[dst_lo]);
 2228     }
 2229     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2230       int vsize = 0;
 2231       switch (ideal_reg()) {
 2232       case Op_VecD:
 2233         vsize = 64;
 2234         break;
 2235       case Op_VecX:
 2236         vsize = 128;
 2237         break;
 2238       case Op_VecA:
 2239         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2240         break;
 2241       default:
 2242         assert(false, "bad register type for spill");
 2243         ShouldNotReachHere();
 2244       }
 2245       st->print("\t# vector spill size = %d", vsize);
 2246     } else if (ideal_reg() == Op_RegVectMask) {
 2247       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2248       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2249       st->print("\t# predicate spill size = %d", vsize);
 2250     } else {
 2251       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2252     }
 2253   }
 2254 
 2255   return 0;
 2256 
 2257 }
 2258 
 2259 #ifndef PRODUCT
 2260 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2261   if (!ra_)
 2262     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2263   else
 2264     implementation(NULL, ra_, false, st);
 2265 }
 2266 #endif
 2267 
 2268 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2269   implementation(&cbuf, ra_, false, NULL);
 2270 }
 2271 
 2272 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2273   return MachNode::size(ra_);
 2274 }
 2275 
 2276 //=============================================================================
 2277 
 2278 #ifndef PRODUCT
 2279 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2280   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2281   int reg = ra_->get_reg_first(this);
 2282   st->print("add %s, rsp, #%d]\t# box lock",
 2283             Matcher::regName[reg], offset);
 2284 }
 2285 #endif
 2286 
 2287 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2288   C2_MacroAssembler _masm(&cbuf);
 2289 
 2290   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2291   int reg    = ra_->get_encode(this);
 2292 
 2293   // This add will handle any 24-bit signed offset. 24 bits allows an
 2294   // 8 megabyte stack frame.
 2295   __ add(as_Register(reg), sp, offset);
 2296 }
 2297 
 2298 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2299   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2300   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2301 
 2302   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2303     return NativeInstruction::instruction_size;
 2304   } else {
 2305     return 2 * NativeInstruction::instruction_size;
 2306   }
 2307 }
 2308 
 2309 ///=============================================================================
 2310 #ifndef PRODUCT
 2311 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2312 {
 2313   st->print_cr("# MachVEPNode");
 2314   if (!_verified) {
 2315     st->print_cr("\t load_class");
 2316   } else {
 2317     st->print_cr("\t unpack_inline_arg");
 2318   }
 2319 }
 2320 #endif
 2321 
 2322 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2323 {
 2324   MacroAssembler _masm(&cbuf);
 2325 
 2326   if (!_verified) {
 2327     Label skip;
 2328     __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2329     __ br(Assembler::EQ, skip);
 2330       __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2331     __ bind(skip);
 2332 
 2333   } else {
 2334     // Unpack inline type args passed as oop and then jump to
 2335     // the verified entry point (skipping the unverified entry).
 2336     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 2337     // Emit code for verified entry and save increment for stack repair on return
 2338     __ verified_entry(ra_->C, sp_inc);
 2339     __ b(*_verified_entry);
 2340   }
 2341 }
 2342 
 2343 //=============================================================================
 2344 #ifndef PRODUCT
 2345 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2346 {
 2347   st->print_cr("# MachUEPNode");
 2348   if (UseCompressedClassPointers) {
 2349     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2350     if (CompressedKlassPointers::shift() != 0) {
 2351       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2352     }
 2353   } else {
 2354    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2355   }
 2356   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2357   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2358 }
 2359 #endif
 2360 
 2361 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2362 {
 2363   // This is the unverified entry point.
 2364   C2_MacroAssembler _masm(&cbuf);
 2365   Label skip;
 2366 
 2367   // UseCompressedClassPointers logic are inside cmp_klass
 2368   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2369 
 2370   // TODO
 2371   // can we avoid this skip and still use a reloc?
 2372   __ br(Assembler::EQ, skip);
 2373   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2374   __ bind(skip);
 2375 }
 2376 
 2377 // REQUIRED EMIT CODE
 2378 
 2379 //=============================================================================
 2380 
 2381 // Emit exception handler code.
 2382 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2383 {
 2384   // mov rscratch1 #exception_blob_entry_point
 2385   // br rscratch1
 2386   // Note that the code buffer's insts_mark is always relative to insts.
 2387   // That's why we must use the macroassembler to generate a handler.
 2388   C2_MacroAssembler _masm(&cbuf);
 2389   address base = __ start_a_stub(size_exception_handler());
 2390   if (base == NULL) {
 2391     ciEnv::current()->record_failure("CodeCache is full");
 2392     return 0;  // CodeBuffer::expand failed
 2393   }
 2394   int offset = __ offset();
 2395   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2396   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2397   __ end_a_stub();
 2398   return offset;
 2399 }
 2400 
 2401 // Emit deopt handler code.
 2402 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2403 {
 2404   // Note that the code buffer's insts_mark is always relative to insts.
 2405   // That's why we must use the macroassembler to generate a handler.
 2406   C2_MacroAssembler _masm(&cbuf);
 2407   address base = __ start_a_stub(size_deopt_handler());
 2408   if (base == NULL) {
 2409     ciEnv::current()->record_failure("CodeCache is full");
 2410     return 0;  // CodeBuffer::expand failed
 2411   }
 2412   int offset = __ offset();
 2413 
 2414   __ adr(lr, __ pc());
 2415   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2416 
 2417   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
 2418   __ end_a_stub();
 2419   return offset;
 2420 }
 2421 
 2422 // REQUIRED MATCHER CODE
 2423 
 2424 //=============================================================================
 2425 
 2426 const bool Matcher::match_rule_supported(int opcode) {
 2427   if (!has_match_rule(opcode))
 2428     return false;
 2429 
 2430   bool ret_value = true;
 2431   switch (opcode) {
 2432     case Op_OnSpinWait:
 2433       return VM_Version::supports_on_spin_wait();
 2434     case Op_CacheWB:
 2435     case Op_CacheWBPreSync:
 2436     case Op_CacheWBPostSync:
 2437       if (!VM_Version::supports_data_cache_line_flush()) {
 2438         ret_value = false;
 2439       }
 2440       break;
 2441     case Op_LoadVectorMasked:
 2442     case Op_StoreVectorMasked:
 2443     case Op_LoadVectorGatherMasked:
 2444     case Op_StoreVectorScatterMasked:
 2445     case Op_MaskAll:
 2446     case Op_AndVMask:
 2447     case Op_OrVMask:
 2448     case Op_XorVMask:
 2449       if (UseSVE == 0) {
 2450         ret_value = false;
 2451       }
 2452       break;
 2453   }
 2454 
 2455   return ret_value; // Per default match rules are supported.
 2456 }
 2457 
 2458 // Identify extra cases that we might want to provide match rules for vector nodes and
 2459 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2460 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2461   if (!match_rule_supported(opcode)) {
 2462     return false;
 2463   }
 2464   int bit_size = vlen * type2aelembytes(bt) * 8;
 2465   if (UseSVE == 0 && bit_size > 128) {
 2466     return false;
 2467   }
 2468   if (UseSVE > 0) {
 2469     return op_sve_supported(opcode, vlen, bt);
 2470   } else { // NEON
 2471     // Special cases
 2472     switch (opcode) {
 2473     case Op_VectorMaskCmp:
 2474       if (vlen < 2 || bit_size < 64) {
 2475         return false;
 2476       }
 2477       break;
 2478     case Op_MulAddVS2VI:
 2479       if (bit_size < 128) {
 2480         return false;
 2481       }
 2482       break;
 2483     case Op_MulVL:
 2484       return false;
 2485     case Op_VectorLoadShuffle:
 2486     case Op_VectorRearrange:
 2487       if (vlen < 4) {
 2488         return false;
 2489       }
 2490       break;
 2491     case Op_LoadVectorGather:
 2492     case Op_StoreVectorScatter:
 2493       return false;
 2494     default:
 2495       break;
 2496     }
 2497   }
 2498   return vector_size_supported(bt, vlen);
 2499 }
 2500 
 2501 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
 2502   // Only SVE supports masked operations.
 2503   if (UseSVE == 0) {
 2504     return false;
 2505   }
 2506   return match_rule_supported(opcode) &&
 2507          masked_op_sve_supported(opcode, vlen, bt);
 2508 }
 2509 
 2510 const RegMask* Matcher::predicate_reg_mask(void) {
 2511   return &_PR_REG_mask;
 2512 }
 2513 
 2514 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2515   return new TypeVectMask(elemTy, length);
 2516 }
 2517 
 2518 // Vector calling convention not yet implemented.
 2519 const bool Matcher::supports_vector_calling_convention(void) {
 2520   return false;
 2521 }
 2522 
 2523 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2524   Unimplemented();
 2525   return OptoRegPair(0, 0);
 2526 }
 2527 
 2528 // Is this branch offset short enough that a short branch can be used?
 2529 //
 2530 // NOTE: If the platform does not provide any short branch variants, then
 2531 //       this method should return false for offset 0.
 2532 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2533   // The passed offset is relative to address of the branch.
 2534 
 2535   return (-32768 <= offset && offset < 32768);
 2536 }
 2537 
 2538 // Vector width in bytes.
 2539 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2540   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2541   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2542   // Minimum 2 values in vector
 2543   if (size < 2*type2aelembytes(bt)) size = 0;
 2544   // But never < 4
 2545   if (size < 4) size = 0;
 2546   return size;
 2547 }
 2548 
 2549 // Limits on vector size (number of elements) loaded into vector.
 2550 const int Matcher::max_vector_size(const BasicType bt) {
 2551   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2552 }
 2553 
 2554 const int Matcher::min_vector_size(const BasicType bt) {
 2555   int max_size = max_vector_size(bt);
 2556   // Limit the min vector size to 8 bytes.
 2557   int size = 8 / type2aelembytes(bt);
 2558   if (bt == T_BYTE) {
 2559     // To support vector api shuffle/rearrange.
 2560     size = 4;
 2561   } else if (bt == T_BOOLEAN) {
 2562     // To support vector api load/store mask.
 2563     size = 2;
 2564   }
 2565   if (size < 2) size = 2;
 2566   return MIN2(size, max_size);
 2567 }
 2568 
 2569 // Actual max scalable vector register length.
 2570 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2571   return Matcher::max_vector_size(bt);
 2572 }
 2573 
 2574 // Vector ideal reg.
 2575 const uint Matcher::vector_ideal_reg(int len) {
 2576   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2577     return Op_VecA;
 2578   }
 2579   switch(len) {
 2580     // For 16-bit/32-bit mask vector, reuse VecD.
 2581     case  2:
 2582     case  4:
 2583     case  8: return Op_VecD;
 2584     case 16: return Op_VecX;
 2585   }
 2586   ShouldNotReachHere();
 2587   return 0;
 2588 }
 2589 
 2590 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2591   ShouldNotReachHere(); // generic vector operands not supported
 2592   return NULL;
 2593 }
 2594 
 2595 bool Matcher::is_reg2reg_move(MachNode* m) {
 2596   ShouldNotReachHere();  // generic vector operands not supported
 2597   return false;
 2598 }
 2599 
 2600 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2601   ShouldNotReachHere();  // generic vector operands not supported
 2602   return false;
 2603 }
 2604 
 2605 // Return whether or not this register is ever used as an argument.
 2606 // This function is used on startup to build the trampoline stubs in
 2607 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2608 // call in the trampoline, and arguments in those registers not be
 2609 // available to the callee.
 2610 bool Matcher::can_be_java_arg(int reg)
 2611 {
 2612   return
 2613     reg ==  R0_num || reg == R0_H_num ||
 2614     reg ==  R1_num || reg == R1_H_num ||
 2615     reg ==  R2_num || reg == R2_H_num ||
 2616     reg ==  R3_num || reg == R3_H_num ||
 2617     reg ==  R4_num || reg == R4_H_num ||
 2618     reg ==  R5_num || reg == R5_H_num ||
 2619     reg ==  R6_num || reg == R6_H_num ||
 2620     reg ==  R7_num || reg == R7_H_num ||
 2621     reg ==  V0_num || reg == V0_H_num ||
 2622     reg ==  V1_num || reg == V1_H_num ||
 2623     reg ==  V2_num || reg == V2_H_num ||
 2624     reg ==  V3_num || reg == V3_H_num ||
 2625     reg ==  V4_num || reg == V4_H_num ||
 2626     reg ==  V5_num || reg == V5_H_num ||
 2627     reg ==  V6_num || reg == V6_H_num ||
 2628     reg ==  V7_num || reg == V7_H_num;
 2629 }
 2630 
 2631 bool Matcher::is_spillable_arg(int reg)
 2632 {
 2633   return can_be_java_arg(reg);
 2634 }
 2635 
 2636 uint Matcher::int_pressure_limit()
 2637 {
 2638   // JDK-8183543: When taking the number of available registers as int
 2639   // register pressure threshold, the jtreg test:
 2640   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2641   // failed due to C2 compilation failure with
 2642   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2643   //
 2644   // A derived pointer is live at CallNode and then is flagged by RA
 2645   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2646   // derived pointers and lastly fail to spill after reaching maximum
 2647   // number of iterations. Lowering the default pressure threshold to
 2648   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2649   // a high register pressure area of the code so that split_DEF can
 2650   // generate DefinitionSpillCopy for the derived pointer.
 2651   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2652   if (!PreserveFramePointer) {
 2653     // When PreserveFramePointer is off, frame pointer is allocatable,
 2654     // but different from other SOC registers, it is excluded from
 2655     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2656     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2657     // See check_pressure_at_fatproj().
 2658     default_int_pressure_threshold--;
 2659   }
 2660   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2661 }
 2662 
 2663 uint Matcher::float_pressure_limit()
 2664 {
 2665   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2666   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2667 }
 2668 
 2669 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2670   return false;
 2671 }
 2672 
 2673 RegMask Matcher::divI_proj_mask() {
 2674   ShouldNotReachHere();
 2675   return RegMask();
 2676 }
 2677 
 2678 // Register for MODI projection of divmodI.
 2679 RegMask Matcher::modI_proj_mask() {
 2680   ShouldNotReachHere();
 2681   return RegMask();
 2682 }
 2683 
 2684 // Register for DIVL projection of divmodL.
 2685 RegMask Matcher::divL_proj_mask() {
 2686   ShouldNotReachHere();
 2687   return RegMask();
 2688 }
 2689 
 2690 // Register for MODL projection of divmodL.
 2691 RegMask Matcher::modL_proj_mask() {
 2692   ShouldNotReachHere();
 2693   return RegMask();
 2694 }
 2695 
 2696 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2697   return FP_REG_mask();
 2698 }
 2699 
 2700 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2701   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2702     Node* u = addp->fast_out(i);
 2703     if (u->is_LoadStore()) {
 2704       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2705       // instructions) only take register indirect as an operand, so
 2706       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2707       // must fail.
 2708       return false;
 2709     }
 2710     if (u->is_Mem()) {
 2711       int opsize = u->as_Mem()->memory_size();
 2712       assert(opsize > 0, "unexpected memory operand size");
 2713       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2714         return false;
 2715       }
 2716     }
 2717   }
 2718   return true;
 2719 }
 2720 
 2721 bool can_combine_with_imm(Node* binary_node, Node* replicate_node) {
 2722   if (UseSVE == 0 || !VectorNode::is_invariant_vector(replicate_node)){
 2723     return false;
 2724   }
 2725   Node* imm_node = replicate_node->in(1);
 2726   if (!imm_node->is_Con()) {
 2727     return false;
 2728   }
 2729 
 2730   const Type* t = imm_node->bottom_type();
 2731   if (!(t->isa_int() || t->isa_long())) {
 2732     return false;
 2733   }
 2734 
 2735   switch (binary_node->Opcode()) {
 2736   case Op_AndV:
 2737   case Op_OrV:
 2738   case Op_XorV: {
 2739     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(binary_node));
 2740     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2741     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2742   }
 2743   case Op_AddVB:
 2744     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2745   case Op_AddVS:
 2746   case Op_AddVI:
 2747     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2748   case Op_AddVL:
 2749     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2750   default:
 2751     return false;
 2752   }
 2753 }
 2754 
 2755 bool is_vector_arith_imm_pattern(Node* n, Node* m) {
 2756   if (n != NULL && m != NULL) {
 2757     return can_combine_with_imm(n, m);
 2758   }
 2759   return false;
 2760 }
 2761 
 2762 // Should the matcher clone input 'm' of node 'n'?
 2763 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2764   // ShiftV src (ShiftCntV con)
 2765   // Binary src (Replicate con)
 2766   if (is_vshift_con_pattern(n, m) || is_vector_arith_imm_pattern(n, m)) {
 2767     mstack.push(m, Visit);
 2768     return true;
 2769   }
 2770 
 2771   return false;
 2772 }
 2773 
 2774 // Should the Matcher clone shifts on addressing modes, expecting them
 2775 // to be subsumed into complex addressing expressions or compute them
 2776 // into registers?
 2777 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2778   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2779     return true;
 2780   }
 2781 
 2782   Node *off = m->in(AddPNode::Offset);
 2783   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2784       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2785       // Are there other uses besides address expressions?
 2786       !is_visited(off)) {
 2787     address_visited.set(off->_idx); // Flag as address_visited
 2788     mstack.push(off->in(2), Visit);
 2789     Node *conv = off->in(1);
 2790     if (conv->Opcode() == Op_ConvI2L &&
 2791         // Are there other uses besides address expressions?
 2792         !is_visited(conv)) {
 2793       address_visited.set(conv->_idx); // Flag as address_visited
 2794       mstack.push(conv->in(1), Pre_Visit);
 2795     } else {
 2796       mstack.push(conv, Pre_Visit);
 2797     }
 2798     address_visited.test_set(m->_idx); // Flag as address_visited
 2799     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2800     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2801     return true;
 2802   } else if (off->Opcode() == Op_ConvI2L &&
 2803              // Are there other uses besides address expressions?
 2804              !is_visited(off)) {
 2805     address_visited.test_set(m->_idx); // Flag as address_visited
 2806     address_visited.set(off->_idx); // Flag as address_visited
 2807     mstack.push(off->in(1), Pre_Visit);
 2808     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2809     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2810     return true;
 2811   }
 2812   return false;
 2813 }
 2814 
 2815 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2816   C2_MacroAssembler _masm(&cbuf);                                       \
 2817   {                                                                     \
 2818     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2819     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2820     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2821     __ INSN(REG, as_Register(BASE));                                    \
 2822   }
 2823 
 2824 
 2825 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2826   {
 2827     Address::extend scale;
 2828 
 2829     // Hooboy, this is fugly.  We need a way to communicate to the
 2830     // encoder that the index needs to be sign extended, so we have to
 2831     // enumerate all the cases.
 2832     switch (opcode) {
 2833     case INDINDEXSCALEDI2L:
 2834     case INDINDEXSCALEDI2LN:
 2835     case INDINDEXI2L:
 2836     case INDINDEXI2LN:
 2837       scale = Address::sxtw(size);
 2838       break;
 2839     default:
 2840       scale = Address::lsl(size);
 2841     }
 2842 
 2843     if (index == -1) {
 2844       return Address(base, disp);
 2845     } else {
 2846       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2847       return Address(base, as_Register(index), scale);
 2848     }
 2849   }
 2850 
 2851 
 2852 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2853 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2854 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2855 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2856                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2857 
 2858   // Used for all non-volatile memory accesses.  The use of
 2859   // $mem->opcode() to discover whether this pattern uses sign-extended
 2860   // offsets is something of a kludge.
 2861   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2862                         Register reg, int opcode,
 2863                         Register base, int index, int scale, int disp,
 2864                         int size_in_memory)
 2865   {
 2866     Address addr = mem2address(opcode, base, index, scale, disp);
 2867     if (addr.getMode() == Address::base_plus_offset) {
 2868       /* If we get an out-of-range offset it is a bug in the compiler,
 2869          so we assert here. */
 2870       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2871              "c2 compiler bug");
 2872       /* Fix up any out-of-range offsets. */
 2873       assert_different_registers(rscratch1, base);
 2874       assert_different_registers(rscratch1, reg);
 2875       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2876     }
 2877     (masm.*insn)(reg, addr);
 2878   }
 2879 
 2880   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2881                         FloatRegister reg, int opcode,
 2882                         Register base, int index, int size, int disp,
 2883                         int size_in_memory)
 2884   {
 2885     Address::extend scale;
 2886 
 2887     switch (opcode) {
 2888     case INDINDEXSCALEDI2L:
 2889     case INDINDEXSCALEDI2LN:
 2890       scale = Address::sxtw(size);
 2891       break;
 2892     default:
 2893       scale = Address::lsl(size);
 2894     }
 2895 
 2896     if (index == -1) {
 2897       /* If we get an out-of-range offset it is a bug in the compiler,
 2898          so we assert here. */
 2899       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2900       /* Fix up any out-of-range offsets. */
 2901       assert_different_registers(rscratch1, base);
 2902       Address addr = Address(base, disp);
 2903       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2904       (masm.*insn)(reg, addr);
 2905     } else {
 2906       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2907       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2908     }
 2909   }
 2910 
 2911   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2912                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2913                         int opcode, Register base, int index, int size, int disp)
 2914   {
 2915     if (index == -1) {
 2916       (masm.*insn)(reg, T, Address(base, disp));
 2917     } else {
 2918       assert(disp == 0, "unsupported address mode");
 2919       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2920     }
 2921   }
 2922 
 2923 %}
 2924 
 2925 
 2926 
 2927 //----------ENCODING BLOCK-----------------------------------------------------
 2928 // This block specifies the encoding classes used by the compiler to
 2929 // output byte streams.  Encoding classes are parameterized macros
 2930 // used by Machine Instruction Nodes in order to generate the bit
 2931 // encoding of the instruction.  Operands specify their base encoding
 2932 // interface with the interface keyword.  There are currently
 2933 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2934 // COND_INTER.  REG_INTER causes an operand to generate a function
 2935 // which returns its register number when queried.  CONST_INTER causes
 2936 // an operand to generate a function which returns the value of the
 2937 // constant when queried.  MEMORY_INTER causes an operand to generate
 2938 // four functions which return the Base Register, the Index Register,
 2939 // the Scale Value, and the Offset Value of the operand when queried.
 2940 // COND_INTER causes an operand to generate six functions which return
 2941 // the encoding code (ie - encoding bits for the instruction)
 2942 // associated with each basic boolean condition for a conditional
 2943 // instruction.
 2944 //
 2945 // Instructions specify two basic values for encoding.  Again, a
 2946 // function is available to check if the constant displacement is an
 2947 // oop. They use the ins_encode keyword to specify their encoding
 2948 // classes (which must be a sequence of enc_class names, and their
 2949 // parameters, specified in the encoding block), and they use the
 2950 // opcode keyword to specify, in order, their primary, secondary, and
 2951 // tertiary opcode.  Only the opcode sections which a particular
 2952 // instruction needs for encoding need to be specified.
 2953 encode %{
 2954   // Build emit functions for each basic byte or larger field in the
 2955   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2956   // from C++ code in the enc_class source block.  Emit functions will
 2957   // live in the main source block for now.  In future, we can
 2958   // generalize this by adding a syntax that specifies the sizes of
 2959   // fields in an order, so that the adlc can build the emit functions
 2960   // automagically
 2961 
 2962   // catch all for unimplemented encodings
 2963   enc_class enc_unimplemented %{
 2964     C2_MacroAssembler _masm(&cbuf);
 2965     __ unimplemented("C2 catch all");
 2966   %}
 2967 
 2968   // BEGIN Non-volatile memory access
 2969 
 2970   // This encoding class is generated automatically from ad_encode.m4.
 2971   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2972   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2973     Register dst_reg = as_Register($dst$$reg);
 2974     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2975                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2976   %}
 2977 
 2978   // This encoding class is generated automatically from ad_encode.m4.
 2979   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2980   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2981     Register dst_reg = as_Register($dst$$reg);
 2982     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2983                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2984   %}
 2985 
 2986   // This encoding class is generated automatically from ad_encode.m4.
 2987   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2988   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2989     Register dst_reg = as_Register($dst$$reg);
 2990     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2991                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2992   %}
 2993 
 2994   // This encoding class is generated automatically from ad_encode.m4.
 2995   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2996   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2997     Register dst_reg = as_Register($dst$$reg);
 2998     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2999                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3000   %}
 3001 
 3002   // This encoding class is generated automatically from ad_encode.m4.
 3003   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3004   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 3005     Register dst_reg = as_Register($dst$$reg);
 3006     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 3007                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3008   %}
 3009 
 3010   // This encoding class is generated automatically from ad_encode.m4.
 3011   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3012   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 3013     Register dst_reg = as_Register($dst$$reg);
 3014     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 3015                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3016   %}
 3017 
 3018   // This encoding class is generated automatically from ad_encode.m4.
 3019   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3020   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 3021     Register dst_reg = as_Register($dst$$reg);
 3022     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3023                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3024   %}
 3025 
 3026   // This encoding class is generated automatically from ad_encode.m4.
 3027   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3028   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 3029     Register dst_reg = as_Register($dst$$reg);
 3030     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3031                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3032   %}
 3033 
 3034   // This encoding class is generated automatically from ad_encode.m4.
 3035   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3036   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 3037     Register dst_reg = as_Register($dst$$reg);
 3038     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3039                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3040   %}
 3041 
 3042   // This encoding class is generated automatically from ad_encode.m4.
 3043   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3044   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 3045     Register dst_reg = as_Register($dst$$reg);
 3046     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3047                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3048   %}
 3049 
 3050   // This encoding class is generated automatically from ad_encode.m4.
 3051   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3052   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 3053     Register dst_reg = as_Register($dst$$reg);
 3054     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 3055                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3056   %}
 3057 
 3058   // This encoding class is generated automatically from ad_encode.m4.
 3059   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3060   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 3061     Register dst_reg = as_Register($dst$$reg);
 3062     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 3063                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3064   %}
 3065 
 3066   // This encoding class is generated automatically from ad_encode.m4.
 3067   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3068   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3069     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3070     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3071                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3072   %}
 3073 
 3074   // This encoding class is generated automatically from ad_encode.m4.
 3075   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3076   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3077     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3078     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3079                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3080   %}
 3081 
 3082   // This encoding class is generated automatically from ad_encode.m4.
 3083   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3084   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3085     Register src_reg = as_Register($src$$reg);
 3086     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 3087                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3088   %}
 3089 
 3090   // This encoding class is generated automatically from ad_encode.m4.
 3091   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3092   enc_class aarch64_enc_strb0(memory1 mem) %{
 3093     C2_MacroAssembler _masm(&cbuf);
 3094     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3095                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3096   %}
 3097 
 3098   // This encoding class is generated automatically from ad_encode.m4.
 3099   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3100   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3101     Register src_reg = as_Register($src$$reg);
 3102     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3103                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3104   %}
 3105 
 3106   // This encoding class is generated automatically from ad_encode.m4.
 3107   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3108   enc_class aarch64_enc_strh0(memory2 mem) %{
 3109     C2_MacroAssembler _masm(&cbuf);
 3110     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3111                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3112   %}
 3113 
 3114   // This encoding class is generated automatically from ad_encode.m4.
 3115   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3116   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3117     Register src_reg = as_Register($src$$reg);
 3118     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3119                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3120   %}
 3121 
 3122   // This encoding class is generated automatically from ad_encode.m4.
 3123   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3124   enc_class aarch64_enc_strw0(memory4 mem) %{
 3125     C2_MacroAssembler _masm(&cbuf);
 3126     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3127                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3128   %}
 3129 
 3130   // This encoding class is generated automatically from ad_encode.m4.
 3131   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3132   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3133     Register src_reg = as_Register($src$$reg);
 3134     // we sometimes get asked to store the stack pointer into the
 3135     // current thread -- we cannot do that directly on AArch64
 3136     if (src_reg == r31_sp) {
 3137       C2_MacroAssembler _masm(&cbuf);
 3138       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3139       __ mov(rscratch2, sp);
 3140       src_reg = rscratch2;
 3141     }
 3142     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3143                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3144   %}
 3145 
 3146   // This encoding class is generated automatically from ad_encode.m4.
 3147   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3148   enc_class aarch64_enc_str0(memory8 mem) %{
 3149     C2_MacroAssembler _masm(&cbuf);
 3150     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3151                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3152   %}
 3153 
 3154   // This encoding class is generated automatically from ad_encode.m4.
 3155   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3156   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3157     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3158     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3159                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3160   %}
 3161 
 3162   // This encoding class is generated automatically from ad_encode.m4.
 3163   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3164   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3165     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3166     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3167                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3168   %}
 3169 
 3170   // This encoding class is generated automatically from ad_encode.m4.
 3171   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3172   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3173       C2_MacroAssembler _masm(&cbuf);
 3174       __ membar(Assembler::StoreStore);
 3175       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3176                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3177   %}
 3178 
 3179   // END Non-volatile memory access
 3180 
 3181   // Vector loads and stores
 3182   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3183     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3184     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3185        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3186   %}
 3187 
 3188   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3189     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3190     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3191        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3192   %}
 3193 
 3194   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3195     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3196     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3197        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3198   %}
 3199 
 3200   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3201     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3202     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3203        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3204   %}
 3205 
 3206   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3207     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3208     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3209        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3210   %}
 3211 
 3212   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3213     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3214     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3215        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3216   %}
 3217 
 3218   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3219     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3220     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3221        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3222   %}
 3223 
 3224   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3225     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3226     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3227        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3228   %}
 3229 
 3230   // volatile loads and stores
 3231 
 3232   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3233     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3234                  rscratch1, stlrb);
 3235   %}
 3236 
 3237   enc_class aarch64_enc_stlrb0(memory mem) %{
 3238     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3239                  rscratch1, stlrb);
 3240   %}
 3241 
 3242   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3243     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3244                  rscratch1, stlrh);
 3245   %}
 3246 
 3247   enc_class aarch64_enc_stlrh0(memory mem) %{
 3248     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3249                  rscratch1, stlrh);
 3250   %}
 3251 
 3252   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3253     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3254                  rscratch1, stlrw);
 3255   %}
 3256 
 3257   enc_class aarch64_enc_stlrw0(memory mem) %{
 3258     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3259                  rscratch1, stlrw);
 3260   %}
 3261 
 3262   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3263     Register dst_reg = as_Register($dst$$reg);
 3264     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3265              rscratch1, ldarb);
 3266     __ sxtbw(dst_reg, dst_reg);
 3267   %}
 3268 
 3269   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3270     Register dst_reg = as_Register($dst$$reg);
 3271     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3272              rscratch1, ldarb);
 3273     __ sxtb(dst_reg, dst_reg);
 3274   %}
 3275 
 3276   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3277     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3278              rscratch1, ldarb);
 3279   %}
 3280 
 3281   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3282     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3283              rscratch1, ldarb);
 3284   %}
 3285 
 3286   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3287     Register dst_reg = as_Register($dst$$reg);
 3288     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3289              rscratch1, ldarh);
 3290     __ sxthw(dst_reg, dst_reg);
 3291   %}
 3292 
 3293   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3294     Register dst_reg = as_Register($dst$$reg);
 3295     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3296              rscratch1, ldarh);
 3297     __ sxth(dst_reg, dst_reg);
 3298   %}
 3299 
 3300   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3301     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3302              rscratch1, ldarh);
 3303   %}
 3304 
 3305   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3306     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3307              rscratch1, ldarh);
 3308   %}
 3309 
 3310   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3311     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3312              rscratch1, ldarw);
 3313   %}
 3314 
 3315   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3316     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3317              rscratch1, ldarw);
 3318   %}
 3319 
 3320   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3321     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3322              rscratch1, ldar);
 3323   %}
 3324 
 3325   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3326     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3327              rscratch1, ldarw);
 3328     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3329   %}
 3330 
 3331   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3332     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3333              rscratch1, ldar);
 3334     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3335   %}
 3336 
 3337   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3338     Register src_reg = as_Register($src$$reg);
 3339     // we sometimes get asked to store the stack pointer into the
 3340     // current thread -- we cannot do that directly on AArch64
 3341     if (src_reg == r31_sp) {
 3342       C2_MacroAssembler _masm(&cbuf);
 3343       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3344       __ mov(rscratch2, sp);
 3345       src_reg = rscratch2;
 3346     }
 3347     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3348                  rscratch1, stlr);
 3349   %}
 3350 
 3351   enc_class aarch64_enc_stlr0(memory mem) %{
 3352     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3353                  rscratch1, stlr);
 3354   %}
 3355 
 3356   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3357     {
 3358       C2_MacroAssembler _masm(&cbuf);
 3359       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3360       __ fmovs(rscratch2, src_reg);
 3361     }
 3362     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3363                  rscratch1, stlrw);
 3364   %}
 3365 
 3366   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3367     {
 3368       C2_MacroAssembler _masm(&cbuf);
 3369       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3370       __ fmovd(rscratch2, src_reg);
 3371     }
 3372     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3373                  rscratch1, stlr);
 3374   %}
 3375 
 3376   // synchronized read/update encodings
 3377 
 3378   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3379     C2_MacroAssembler _masm(&cbuf);
 3380     Register dst_reg = as_Register($dst$$reg);
 3381     Register base = as_Register($mem$$base);
 3382     int index = $mem$$index;
 3383     int scale = $mem$$scale;
 3384     int disp = $mem$$disp;
 3385     if (index == -1) {
 3386        if (disp != 0) {
 3387         __ lea(rscratch1, Address(base, disp));
 3388         __ ldaxr(dst_reg, rscratch1);
 3389       } else {
 3390         // TODO
 3391         // should we ever get anything other than this case?
 3392         __ ldaxr(dst_reg, base);
 3393       }
 3394     } else {
 3395       Register index_reg = as_Register(index);
 3396       if (disp == 0) {
 3397         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3398         __ ldaxr(dst_reg, rscratch1);
 3399       } else {
 3400         __ lea(rscratch1, Address(base, disp));
 3401         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3402         __ ldaxr(dst_reg, rscratch1);
 3403       }
 3404     }
 3405   %}
 3406 
 3407   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3408     C2_MacroAssembler _masm(&cbuf);
 3409     Register src_reg = as_Register($src$$reg);
 3410     Register base = as_Register($mem$$base);
 3411     int index = $mem$$index;
 3412     int scale = $mem$$scale;
 3413     int disp = $mem$$disp;
 3414     if (index == -1) {
 3415        if (disp != 0) {
 3416         __ lea(rscratch2, Address(base, disp));
 3417         __ stlxr(rscratch1, src_reg, rscratch2);
 3418       } else {
 3419         // TODO
 3420         // should we ever get anything other than this case?
 3421         __ stlxr(rscratch1, src_reg, base);
 3422       }
 3423     } else {
 3424       Register index_reg = as_Register(index);
 3425       if (disp == 0) {
 3426         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3427         __ stlxr(rscratch1, src_reg, rscratch2);
 3428       } else {
 3429         __ lea(rscratch2, Address(base, disp));
 3430         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3431         __ stlxr(rscratch1, src_reg, rscratch2);
 3432       }
 3433     }
 3434     __ cmpw(rscratch1, zr);
 3435   %}
 3436 
 3437   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3438     C2_MacroAssembler _masm(&cbuf);
 3439     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3440     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3441                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3442                /*weak*/ false, noreg);
 3443   %}
 3444 
 3445   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3446     C2_MacroAssembler _masm(&cbuf);
 3447     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3448     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3449                Assembler::word, /*acquire*/ false, /*release*/ true,
 3450                /*weak*/ false, noreg);
 3451   %}
 3452 
 3453   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3454     C2_MacroAssembler _masm(&cbuf);
 3455     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3456     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3457                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3458                /*weak*/ false, noreg);
 3459   %}
 3460 
 3461   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3462     C2_MacroAssembler _masm(&cbuf);
 3463     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3464     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3465                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3466                /*weak*/ false, noreg);
 3467   %}
 3468 
 3469 
 3470   // The only difference between aarch64_enc_cmpxchg and
 3471   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3472   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3473   // lock.
 3474   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3475     C2_MacroAssembler _masm(&cbuf);
 3476     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3477     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3478                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3479                /*weak*/ false, noreg);
 3480   %}
 3481 
 3482   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3483     C2_MacroAssembler _masm(&cbuf);
 3484     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3485     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3486                Assembler::word, /*acquire*/ true, /*release*/ true,
 3487                /*weak*/ false, noreg);
 3488   %}
 3489 
 3490   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3491     C2_MacroAssembler _masm(&cbuf);
 3492     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3493     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3494                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3495                /*weak*/ false, noreg);
 3496   %}
 3497 
 3498   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3499     C2_MacroAssembler _masm(&cbuf);
 3500     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3501     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3502                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3503                /*weak*/ false, noreg);
 3504   %}
 3505 
 3506   // auxiliary used for CompareAndSwapX to set result register
 3507   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3508     C2_MacroAssembler _masm(&cbuf);
 3509     Register res_reg = as_Register($res$$reg);
 3510     __ cset(res_reg, Assembler::EQ);
 3511   %}
 3512 
 3513   // prefetch encodings
 3514 
 3515   enc_class aarch64_enc_prefetchw(memory mem) %{
 3516     C2_MacroAssembler _masm(&cbuf);
 3517     Register base = as_Register($mem$$base);
 3518     int index = $mem$$index;
 3519     int scale = $mem$$scale;
 3520     int disp = $mem$$disp;
 3521     if (index == -1) {
 3522       __ prfm(Address(base, disp), PSTL1KEEP);
 3523     } else {
 3524       Register index_reg = as_Register(index);
 3525       if (disp == 0) {
 3526         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3527       } else {
 3528         __ lea(rscratch1, Address(base, disp));
 3529 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3530       }
 3531     }
 3532   %}
 3533 
 3534   /// mov envcodings
 3535 
 3536   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3537     C2_MacroAssembler _masm(&cbuf);
 3538     uint32_t con = (uint32_t)$src$$constant;
 3539     Register dst_reg = as_Register($dst$$reg);
 3540     if (con == 0) {
 3541       __ movw(dst_reg, zr);
 3542     } else {
 3543       __ movw(dst_reg, con);
 3544     }
 3545   %}
 3546 
 3547   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3548     C2_MacroAssembler _masm(&cbuf);
 3549     Register dst_reg = as_Register($dst$$reg);
 3550     uint64_t con = (uint64_t)$src$$constant;
 3551     if (con == 0) {
 3552       __ mov(dst_reg, zr);
 3553     } else {
 3554       __ mov(dst_reg, con);
 3555     }
 3556   %}
 3557 
 3558   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3559     C2_MacroAssembler _masm(&cbuf);
 3560     Register dst_reg = as_Register($dst$$reg);
 3561     address con = (address)$src$$constant;
 3562     if (con == NULL || con == (address)1) {
 3563       ShouldNotReachHere();
 3564     } else {
 3565       relocInfo::relocType rtype = $src->constant_reloc();
 3566       if (rtype == relocInfo::oop_type) {
 3567         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3568       } else if (rtype == relocInfo::metadata_type) {
 3569         __ mov_metadata(dst_reg, (Metadata*)con);
 3570       } else {
 3571         assert(rtype == relocInfo::none, "unexpected reloc type");
 3572         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3573           __ mov(dst_reg, con);
 3574         } else {
 3575           uint64_t offset;
 3576           __ adrp(dst_reg, con, offset);
 3577           __ add(dst_reg, dst_reg, offset);
 3578         }
 3579       }
 3580     }
 3581   %}
 3582 
 3583   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3584     C2_MacroAssembler _masm(&cbuf);
 3585     Register dst_reg = as_Register($dst$$reg);
 3586     __ mov(dst_reg, zr);
 3587   %}
 3588 
 3589   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3590     C2_MacroAssembler _masm(&cbuf);
 3591     Register dst_reg = as_Register($dst$$reg);
 3592     __ mov(dst_reg, (uint64_t)1);
 3593   %}
 3594 
 3595   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3596     C2_MacroAssembler _masm(&cbuf);
 3597     __ load_byte_map_base($dst$$Register);
 3598   %}
 3599 
 3600   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3601     C2_MacroAssembler _masm(&cbuf);
 3602     Register dst_reg = as_Register($dst$$reg);
 3603     address con = (address)$src$$constant;
 3604     if (con == NULL) {
 3605       ShouldNotReachHere();
 3606     } else {
 3607       relocInfo::relocType rtype = $src->constant_reloc();
 3608       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3609       __ set_narrow_oop(dst_reg, (jobject)con);
 3610     }
 3611   %}
 3612 
 3613   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3614     C2_MacroAssembler _masm(&cbuf);
 3615     Register dst_reg = as_Register($dst$$reg);
 3616     __ mov(dst_reg, zr);
 3617   %}
 3618 
 3619   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3620     C2_MacroAssembler _masm(&cbuf);
 3621     Register dst_reg = as_Register($dst$$reg);
 3622     address con = (address)$src$$constant;
 3623     if (con == NULL) {
 3624       ShouldNotReachHere();
 3625     } else {
 3626       relocInfo::relocType rtype = $src->constant_reloc();
 3627       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3628       __ set_narrow_klass(dst_reg, (Klass *)con);
 3629     }
 3630   %}
 3631 
 3632   // arithmetic encodings
 3633 
 3634   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3635     C2_MacroAssembler _masm(&cbuf);
 3636     Register dst_reg = as_Register($dst$$reg);
 3637     Register src_reg = as_Register($src1$$reg);
 3638     int32_t con = (int32_t)$src2$$constant;
 3639     // add has primary == 0, subtract has primary == 1
 3640     if ($primary) { con = -con; }
 3641     if (con < 0) {
 3642       __ subw(dst_reg, src_reg, -con);
 3643     } else {
 3644       __ addw(dst_reg, src_reg, con);
 3645     }
 3646   %}
 3647 
 3648   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3649     C2_MacroAssembler _masm(&cbuf);
 3650     Register dst_reg = as_Register($dst$$reg);
 3651     Register src_reg = as_Register($src1$$reg);
 3652     int32_t con = (int32_t)$src2$$constant;
 3653     // add has primary == 0, subtract has primary == 1
 3654     if ($primary) { con = -con; }
 3655     if (con < 0) {
 3656       __ sub(dst_reg, src_reg, -con);
 3657     } else {
 3658       __ add(dst_reg, src_reg, con);
 3659     }
 3660   %}
 3661 
 3662   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3663     C2_MacroAssembler _masm(&cbuf);
 3664    Register dst_reg = as_Register($dst$$reg);
 3665    Register src1_reg = as_Register($src1$$reg);
 3666    Register src2_reg = as_Register($src2$$reg);
 3667     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3668   %}
 3669 
 3670   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3671     C2_MacroAssembler _masm(&cbuf);
 3672    Register dst_reg = as_Register($dst$$reg);
 3673    Register src1_reg = as_Register($src1$$reg);
 3674    Register src2_reg = as_Register($src2$$reg);
 3675     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3676   %}
 3677 
 3678   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3679     C2_MacroAssembler _masm(&cbuf);
 3680    Register dst_reg = as_Register($dst$$reg);
 3681    Register src1_reg = as_Register($src1$$reg);
 3682    Register src2_reg = as_Register($src2$$reg);
 3683     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3684   %}
 3685 
 3686   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3687     C2_MacroAssembler _masm(&cbuf);
 3688    Register dst_reg = as_Register($dst$$reg);
 3689    Register src1_reg = as_Register($src1$$reg);
 3690    Register src2_reg = as_Register($src2$$reg);
 3691     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3692   %}
 3693 
 3694   // compare instruction encodings
 3695 
 3696   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3697     C2_MacroAssembler _masm(&cbuf);
 3698     Register reg1 = as_Register($src1$$reg);
 3699     Register reg2 = as_Register($src2$$reg);
 3700     __ cmpw(reg1, reg2);
 3701   %}
 3702 
 3703   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3704     C2_MacroAssembler _masm(&cbuf);
 3705     Register reg = as_Register($src1$$reg);
 3706     int32_t val = $src2$$constant;
 3707     if (val >= 0) {
 3708       __ subsw(zr, reg, val);
 3709     } else {
 3710       __ addsw(zr, reg, -val);
 3711     }
 3712   %}
 3713 
 3714   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3715     C2_MacroAssembler _masm(&cbuf);
 3716     Register reg1 = as_Register($src1$$reg);
 3717     uint32_t val = (uint32_t)$src2$$constant;
 3718     __ movw(rscratch1, val);
 3719     __ cmpw(reg1, rscratch1);
 3720   %}
 3721 
 3722   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3723     C2_MacroAssembler _masm(&cbuf);
 3724     Register reg1 = as_Register($src1$$reg);
 3725     Register reg2 = as_Register($src2$$reg);
 3726     __ cmp(reg1, reg2);
 3727   %}
 3728 
 3729   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3730     C2_MacroAssembler _masm(&cbuf);
 3731     Register reg = as_Register($src1$$reg);
 3732     int64_t val = $src2$$constant;
 3733     if (val >= 0) {
 3734       __ subs(zr, reg, val);
 3735     } else if (val != -val) {
 3736       __ adds(zr, reg, -val);
 3737     } else {
 3738     // aargh, Long.MIN_VALUE is a special case
 3739       __ orr(rscratch1, zr, (uint64_t)val);
 3740       __ subs(zr, reg, rscratch1);
 3741     }
 3742   %}
 3743 
 3744   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3745     C2_MacroAssembler _masm(&cbuf);
 3746     Register reg1 = as_Register($src1$$reg);
 3747     uint64_t val = (uint64_t)$src2$$constant;
 3748     __ mov(rscratch1, val);
 3749     __ cmp(reg1, rscratch1);
 3750   %}
 3751 
 3752   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3753     C2_MacroAssembler _masm(&cbuf);
 3754     Register reg1 = as_Register($src1$$reg);
 3755     Register reg2 = as_Register($src2$$reg);
 3756     __ cmp(reg1, reg2);
 3757   %}
 3758 
 3759   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3760     C2_MacroAssembler _masm(&cbuf);
 3761     Register reg1 = as_Register($src1$$reg);
 3762     Register reg2 = as_Register($src2$$reg);
 3763     __ cmpw(reg1, reg2);
 3764   %}
 3765 
 3766   enc_class aarch64_enc_testp(iRegP src) %{
 3767     C2_MacroAssembler _masm(&cbuf);
 3768     Register reg = as_Register($src$$reg);
 3769     __ cmp(reg, zr);
 3770   %}
 3771 
 3772   enc_class aarch64_enc_testn(iRegN src) %{
 3773     C2_MacroAssembler _masm(&cbuf);
 3774     Register reg = as_Register($src$$reg);
 3775     __ cmpw(reg, zr);
 3776   %}
 3777 
 3778   enc_class aarch64_enc_b(label lbl) %{
 3779     C2_MacroAssembler _masm(&cbuf);
 3780     Label *L = $lbl$$label;
 3781     __ b(*L);
 3782   %}
 3783 
 3784   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3785     C2_MacroAssembler _masm(&cbuf);
 3786     Label *L = $lbl$$label;
 3787     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3788   %}
 3789 
 3790   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3791     C2_MacroAssembler _masm(&cbuf);
 3792     Label *L = $lbl$$label;
 3793     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3794   %}
 3795 
 3796   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3797   %{
 3798      Register sub_reg = as_Register($sub$$reg);
 3799      Register super_reg = as_Register($super$$reg);
 3800      Register temp_reg = as_Register($temp$$reg);
 3801      Register result_reg = as_Register($result$$reg);
 3802 
 3803      Label miss;
 3804      C2_MacroAssembler _masm(&cbuf);
 3805      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3806                                      NULL, &miss,
 3807                                      /*set_cond_codes:*/ true);
 3808      if ($primary) {
 3809        __ mov(result_reg, zr);
 3810      }
 3811      __ bind(miss);
 3812   %}
 3813 
 3814   enc_class aarch64_enc_java_static_call(method meth) %{
 3815     C2_MacroAssembler _masm(&cbuf);
 3816 
 3817     address addr = (address)$meth$$method;
 3818     address call;
 3819     if (!_method) {
 3820       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3821       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3822       if (call == NULL) {
 3823         ciEnv::current()->record_failure("CodeCache is full");
 3824         return;
 3825       }
 3826     } else {
 3827       int method_index = resolved_method_index(cbuf);
 3828       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3829                                                   : static_call_Relocation::spec(method_index);
 3830       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3831       if (call == NULL) {
 3832         ciEnv::current()->record_failure("CodeCache is full");
 3833         return;
 3834       }
 3835       // Emit stub for static call
 3836       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3837       if (stub == NULL) {
 3838         ciEnv::current()->record_failure("CodeCache is full");
 3839         return;
 3840       }
 3841     }
 3842 
 3843     // Only non uncommon_trap calls need to reinitialize ptrue.
 3844     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3845       __ reinitialize_ptrue();
 3846     }
 3847   %}
 3848 
 3849   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3850     C2_MacroAssembler _masm(&cbuf);
 3851     int method_index = resolved_method_index(cbuf);
 3852     address call = __ ic_call((address)$meth$$method, method_index);
 3853     if (call == NULL) {
 3854       ciEnv::current()->record_failure("CodeCache is full");
 3855       return;
 3856     } else if (Compile::current()->max_vector_size() > 0) {
 3857       __ reinitialize_ptrue();
 3858     }
 3859   %}
 3860 
 3861   enc_class aarch64_enc_call_epilog() %{
 3862     C2_MacroAssembler _masm(&cbuf);
 3863     if (VerifyStackAtCalls) {
 3864       // Check that stack depth is unchanged: find majik cookie on stack
 3865       __ call_Unimplemented();
 3866     }
 3867     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
 3868       if (!_method->signature()->returns_null_free_inline_type()) {
 3869         // The last return value is not set by the callee but used to pass IsInit information to compiled code.
 3870         // Search for the corresponding projection, get the register and emit code that initialized it.
 3871         uint con = (tf()->range_cc()->cnt() - 1);
 3872         for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
 3873           ProjNode* proj = fast_out(i)->as_Proj();
 3874           if (proj->_con == con) {
 3875             // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
 3876             OptoReg::Name optoReg = ra_->get_reg_first(proj);
 3877             VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
 3878             Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
 3879             __ cmp(r0, zr);
 3880             __ cset(toReg, Assembler::NE);
 3881             if (reg->is_stack()) {
 3882               int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
 3883               __ str(toReg, Address(sp, st_off));
 3884             }
 3885             break;
 3886           }
 3887         }
 3888       }
 3889       if (return_value_is_used()) {
 3890         // An inline type is returned as fields in multiple registers.
 3891         // R0 either contains an oop if the inline type is buffered or a pointer
 3892         // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
 3893         // if the lowest bit is set to allow C2 to use the oop after null checking.
 3894         // r0 &= (r0 & 1) - 1
 3895         __ andr(rscratch1, r0, 0x1);
 3896         __ sub(rscratch1, rscratch1, 0x1);
 3897         __ andr(r0, r0, rscratch1);
 3898       }
 3899     }
 3900   %}
 3901 
 3902   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3903     C2_MacroAssembler _masm(&cbuf);
 3904 
 3905     // some calls to generated routines (arraycopy code) are scheduled
 3906     // by C2 as runtime calls. if so we can call them using a br (they
 3907     // will be in a reachable segment) otherwise we have to use a blr
 3908     // which loads the absolute address into a register.
 3909     address entry = (address)$meth$$method;
 3910     CodeBlob *cb = CodeCache::find_blob(entry);
 3911     if (cb) {
 3912       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3913       if (call == NULL) {
 3914         ciEnv::current()->record_failure("CodeCache is full");
 3915         return;
 3916       }
 3917     } else {
 3918       Label retaddr;
 3919       __ adr(rscratch2, retaddr);
 3920       __ lea(rscratch1, RuntimeAddress(entry));
 3921       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3922       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3923       __ blr(rscratch1);
 3924       __ bind(retaddr);
 3925       __ add(sp, sp, 2 * wordSize);
 3926     }
 3927     if (Compile::current()->max_vector_size() > 0) {
 3928       __ reinitialize_ptrue();
 3929     }
 3930   %}
 3931 
 3932   enc_class aarch64_enc_rethrow() %{
 3933     C2_MacroAssembler _masm(&cbuf);
 3934     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3935   %}
 3936 
 3937   enc_class aarch64_enc_ret() %{
 3938     C2_MacroAssembler _masm(&cbuf);
 3939 #ifdef ASSERT
 3940     if (Compile::current()->max_vector_size() > 0) {
 3941       __ verify_ptrue();
 3942     }
 3943 #endif
 3944     __ ret(lr);
 3945   %}
 3946 
 3947   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3948     C2_MacroAssembler _masm(&cbuf);
 3949     Register target_reg = as_Register($jump_target$$reg);
 3950     __ br(target_reg);
 3951   %}
 3952 
 3953   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3954     C2_MacroAssembler _masm(&cbuf);
 3955     Register target_reg = as_Register($jump_target$$reg);
 3956     // exception oop should be in r0
 3957     // ret addr has been popped into lr
 3958     // callee expects it in r3
 3959     __ mov(r3, lr);
 3960     __ br(target_reg);
 3961   %}
 3962 
 3963   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3964     C2_MacroAssembler _masm(&cbuf);
 3965     Register oop = as_Register($object$$reg);
 3966     Register box = as_Register($box$$reg);
 3967     Register disp_hdr = as_Register($tmp$$reg);
 3968     Register tmp = as_Register($tmp2$$reg);
 3969     Label cont;
 3970     Label object_has_monitor;
 3971     Label cas_failed;
 3972 
 3973     assert_different_registers(oop, box, tmp, disp_hdr);
 3974 
 3975     // Load markWord from object into displaced_header.
 3976     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3977 
 3978     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3979       __ load_klass(tmp, oop);
 3980       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3981       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3982       __ br(Assembler::NE, cont);
 3983     }
 3984 
 3985     // Check for existing monitor
 3986     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3987 
 3988     if (!UseHeavyMonitors) {
 3989       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3990       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3991 
 3992       if (EnableValhalla) {
 3993         // Mask inline_type bit such that we go to the slow path if object is an inline type
 3994         __ andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
 3995       }
 3996 
 3997       // Initialize the box. (Must happen before we update the object mark!)
 3998       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3999 
 4000       // Compare object markWord with an unlocked value (tmp) and if
 4001       // equal exchange the stack address of our box with object markWord.
 4002       // On failure disp_hdr contains the possibly locked markWord.
 4003       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 4004                  /*release*/ true, /*weak*/ false, disp_hdr);
 4005       __ br(Assembler::EQ, cont);
 4006 
 4007       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 4008 
 4009       // If the compare-and-exchange succeeded, then we found an unlocked
 4010       // object, will have now locked it will continue at label cont
 4011 
 4012       __ bind(cas_failed);
 4013       // We did not see an unlocked object so try the fast recursive case.
 4014 
 4015       // Check if the owner is self by comparing the value in the
 4016       // markWord of object (disp_hdr) with the stack pointer.
 4017       __ mov(rscratch1, sp);
 4018       __ sub(disp_hdr, disp_hdr, rscratch1);
 4019       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 4020       // If condition is true we are cont and hence we can store 0 as the
 4021       // displaced header in the box, which indicates that it is a recursive lock.
 4022       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 4023       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 4024     } else {
 4025       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 4026     }
 4027     __ b(cont);
 4028 
 4029     // Handle existing monitor.
 4030     __ bind(object_has_monitor);
 4031 
 4032     // The object's monitor m is unlocked iff m->owner == NULL,
 4033     // otherwise m->owner may contain a thread or a stack address.
 4034     //
 4035     // Try to CAS m->owner from NULL to current thread.
 4036     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 4037     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 4038                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 4039 
 4040     // Store a non-null value into the box to avoid looking like a re-entrant
 4041     // lock. The fast-path monitor unlock code checks for
 4042     // markWord::monitor_value so use markWord::unused_mark which has the
 4043     // relevant bit set, and also matches ObjectSynchronizer::enter.
 4044     __ mov(tmp, (address)markWord::unused_mark().value());
 4045     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 4046 
 4047     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 4048 
 4049     __ cmp(rscratch1, rthread);
 4050     __ br(Assembler::NE, cont); // Check for recursive locking
 4051 
 4052     // Recursive lock case
 4053     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 4054     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 4055 
 4056     __ bind(cont);
 4057     // flag == EQ indicates success
 4058     // flag == NE indicates failure
 4059   %}
 4060 
 4061   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 4062     C2_MacroAssembler _masm(&cbuf);
 4063     Register oop = as_Register($object$$reg);
 4064     Register box = as_Register($box$$reg);
 4065     Register disp_hdr = as_Register($tmp$$reg);
 4066     Register tmp = as_Register($tmp2$$reg);
 4067     Label cont;
 4068     Label object_has_monitor;
 4069 
 4070     assert_different_registers(oop, box, tmp, disp_hdr);
 4071 
 4072     if (!UseHeavyMonitors) {
 4073       // Find the lock address and load the displaced header from the stack.
 4074       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 4075 
 4076       // If the displaced header is 0, we have a recursive unlock.
 4077       __ cmp(disp_hdr, zr);
 4078       __ br(Assembler::EQ, cont);
 4079     }
 4080 
 4081     // Handle existing monitor.
 4082     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 4083     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 4084 
 4085     if (!UseHeavyMonitors) {
 4086       // Check if it is still a light weight lock, this is is true if we
 4087       // see the stack address of the basicLock in the markWord of the
 4088       // object.
 4089 
 4090       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 4091                  /*release*/ true, /*weak*/ false, tmp);
 4092     } else {
 4093       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 4094     }
 4095     __ b(cont);
 4096 
 4097     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 4098 
 4099     // Handle existing monitor.
 4100     __ bind(object_has_monitor);
 4101     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 4102     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 4103     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4104 
 4105     Label notRecursive;
 4106     __ cbz(disp_hdr, notRecursive);
 4107 
 4108     // Recursive lock
 4109     __ sub(disp_hdr, disp_hdr, 1u);
 4110     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4111     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 4112     __ b(cont);
 4113 
 4114     __ bind(notRecursive);
 4115     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 4116     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 4117     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 4118     __ cmp(rscratch1, zr); // Sets flags for result
 4119     __ cbnz(rscratch1, cont);
 4120     // need a release store here
 4121     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 4122     __ stlr(zr, tmp); // set unowned
 4123 
 4124     __ bind(cont);
 4125     // flag == EQ indicates success
 4126     // flag == NE indicates failure
 4127   %}
 4128 
 4129 %}
 4130 
 4131 //----------FRAME--------------------------------------------------------------
 4132 // Definition of frame structure and management information.
 4133 //
 4134 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4135 //                             |   (to get allocators register number
 4136 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4137 //  r   CALLER     |        |
 4138 //  o     |        +--------+      pad to even-align allocators stack-slot
 4139 //  w     V        |  pad0  |        numbers; owned by CALLER
 4140 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4141 //  h     ^        |   in   |  5
 4142 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4143 //  |     |        |        |  3
 4144 //  |     |        +--------+
 4145 //  V     |        | old out|      Empty on Intel, window on Sparc
 4146 //        |    old |preserve|      Must be even aligned.
 4147 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4148 //        |        |   in   |  3   area for Intel ret address
 4149 //     Owned by    |preserve|      Empty on Sparc.
 4150 //       SELF      +--------+
 4151 //        |        |  pad2  |  2   pad to align old SP
 4152 //        |        +--------+  1
 4153 //        |        | locks  |  0
 4154 //        |        +--------+----> OptoReg::stack0(), even aligned
 4155 //        |        |  pad1  | 11   pad to align new SP
 4156 //        |        +--------+
 4157 //        |        |        | 10
 4158 //        |        | spills |  9   spills
 4159 //        V        |        |  8   (pad0 slot for callee)
 4160 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4161 //        ^        |  out   |  7
 4162 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4163 //     Owned by    +--------+
 4164 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4165 //        |    new |preserve|      Must be even-aligned.
 4166 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4167 //        |        |        |
 4168 //
 4169 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4170 //         known from SELF's arguments and the Java calling convention.
 4171 //         Region 6-7 is determined per call site.
 4172 // Note 2: If the calling convention leaves holes in the incoming argument
 4173 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4174 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4175 //         incoming area, as the Java calling convention is completely under
 4176 //         the control of the AD file.  Doubles can be sorted and packed to
 4177 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4178 //         varargs C calling conventions.
 4179 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4180 //         even aligned with pad0 as needed.
 4181 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4182 //           (the latter is true on Intel but is it false on AArch64?)
 4183 //         region 6-11 is even aligned; it may be padded out more so that
 4184 //         the region from SP to FP meets the minimum stack alignment.
 4185 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4186 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4187 //         SP meets the minimum alignment.
 4188 
 4189 frame %{
 4190   // These three registers define part of the calling convention
 4191   // between compiled code and the interpreter.
 4192 
 4193   // Inline Cache Register or Method for I2C.
 4194   inline_cache_reg(R12);
 4195 
 4196   // Number of stack slots consumed by locking an object
 4197   sync_stack_slots(2);
 4198 
 4199   // Compiled code's Frame Pointer
 4200   frame_pointer(R31);
 4201 
 4202   // Interpreter stores its frame pointer in a register which is
 4203   // stored to the stack by I2CAdaptors.
 4204   // I2CAdaptors convert from interpreted java to compiled java.
 4205   interpreter_frame_pointer(R29);
 4206 
 4207   // Stack alignment requirement
 4208   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4209 
 4210   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4211   // for calls to C.  Supports the var-args backing area for register parms.
 4212   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4213 
 4214   // The after-PROLOG location of the return address.  Location of
 4215   // return address specifies a type (REG or STACK) and a number
 4216   // representing the register number (i.e. - use a register name) or
 4217   // stack slot.
 4218   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4219   // Otherwise, it is above the locks and verification slot and alignment word
 4220   // TODO this may well be correct but need to check why that - 2 is there
 4221   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4222   // which folds in the space used for monitors
 4223   return_addr(STACK - 2 +
 4224               align_up((Compile::current()->in_preserve_stack_slots() +
 4225                         Compile::current()->fixed_slots()),
 4226                        stack_alignment_in_slots()));
 4227 
 4228   // Location of compiled Java return values.  Same as C for now.
 4229   return_value
 4230   %{
 4231     // TODO do we allow ideal_reg == Op_RegN???
 4232     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4233            "only return normal values");
 4234 
 4235     static const int lo[Op_RegL + 1] = { // enum name
 4236       0,                                 // Op_Node
 4237       0,                                 // Op_Set
 4238       R0_num,                            // Op_RegN
 4239       R0_num,                            // Op_RegI
 4240       R0_num,                            // Op_RegP
 4241       V0_num,                            // Op_RegF
 4242       V0_num,                            // Op_RegD
 4243       R0_num                             // Op_RegL
 4244     };
 4245 
 4246     static const int hi[Op_RegL + 1] = { // enum name
 4247       0,                                 // Op_Node
 4248       0,                                 // Op_Set
 4249       OptoReg::Bad,                      // Op_RegN
 4250       OptoReg::Bad,                      // Op_RegI
 4251       R0_H_num,                          // Op_RegP
 4252       OptoReg::Bad,                      // Op_RegF
 4253       V0_H_num,                          // Op_RegD
 4254       R0_H_num                           // Op_RegL
 4255     };
 4256 
 4257     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4258   %}
 4259 %}
 4260 
 4261 //----------ATTRIBUTES---------------------------------------------------------
 4262 //----------Operand Attributes-------------------------------------------------
 4263 op_attrib op_cost(1);        // Required cost attribute
 4264 
 4265 //----------Instruction Attributes---------------------------------------------
 4266 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4267 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4268 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4269                                 // a non-matching short branch variant
 4270                                 // of some long branch?
 4271 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4272                                 // be a power of 2) specifies the
 4273                                 // alignment that some part of the
 4274                                 // instruction (not necessarily the
 4275                                 // start) requires.  If > 1, a
 4276                                 // compute_padding() function must be
 4277                                 // provided for the instruction
 4278 
 4279 //----------OPERANDS-----------------------------------------------------------
 4280 // Operand definitions must precede instruction definitions for correct parsing
 4281 // in the ADLC because operands constitute user defined types which are used in
 4282 // instruction definitions.
 4283 
 4284 //----------Simple Operands----------------------------------------------------
 4285 
 4286 // Integer operands 32 bit
 4287 // 32 bit immediate
 4288 operand immI()
 4289 %{
 4290   match(ConI);
 4291 
 4292   op_cost(0);
 4293   format %{ %}
 4294   interface(CONST_INTER);
 4295 %}
 4296 
 4297 // 32 bit zero
 4298 operand immI0()
 4299 %{
 4300   predicate(n->get_int() == 0);
 4301   match(ConI);
 4302 
 4303   op_cost(0);
 4304   format %{ %}
 4305   interface(CONST_INTER);
 4306 %}
 4307 
 4308 // 32 bit unit increment
 4309 operand immI_1()
 4310 %{
 4311   predicate(n->get_int() == 1);
 4312   match(ConI);
 4313 
 4314   op_cost(0);
 4315   format %{ %}
 4316   interface(CONST_INTER);
 4317 %}
 4318 
 4319 // 32 bit unit decrement
 4320 operand immI_M1()
 4321 %{
 4322   predicate(n->get_int() == -1);
 4323   match(ConI);
 4324 
 4325   op_cost(0);
 4326   format %{ %}
 4327   interface(CONST_INTER);
 4328 %}
 4329 
 4330 // Shift values for add/sub extension shift
 4331 operand immIExt()
 4332 %{
 4333   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4334   match(ConI);
 4335 
 4336   op_cost(0);
 4337   format %{ %}
 4338   interface(CONST_INTER);
 4339 %}
 4340 
 4341 operand immI_gt_1()
 4342 %{
 4343   predicate(n->get_int() > 1);
 4344   match(ConI);
 4345 
 4346   op_cost(0);
 4347   format %{ %}
 4348   interface(CONST_INTER);
 4349 %}
 4350 
 4351 operand immI_le_4()
 4352 %{
 4353   predicate(n->get_int() <= 4);
 4354   match(ConI);
 4355 
 4356   op_cost(0);
 4357   format %{ %}
 4358   interface(CONST_INTER);
 4359 %}
 4360 
 4361 operand immI_31()
 4362 %{
 4363   predicate(n->get_int() == 31);
 4364   match(ConI);
 4365 
 4366   op_cost(0);
 4367   format %{ %}
 4368   interface(CONST_INTER);
 4369 %}
 4370 
 4371 operand immI_2()
 4372 %{
 4373   predicate(n->get_int() == 2);
 4374   match(ConI);
 4375 
 4376   op_cost(0);
 4377   format %{ %}
 4378   interface(CONST_INTER);
 4379 %}
 4380 
 4381 operand immI_4()
 4382 %{
 4383   predicate(n->get_int() == 4);
 4384   match(ConI);
 4385 
 4386   op_cost(0);
 4387   format %{ %}
 4388   interface(CONST_INTER);
 4389 %}
 4390 
 4391 operand immI_8()
 4392 %{
 4393   predicate(n->get_int() == 8);
 4394   match(ConI);
 4395 
 4396   op_cost(0);
 4397   format %{ %}
 4398   interface(CONST_INTER);
 4399 %}
 4400 
 4401 operand immI_16()
 4402 %{
 4403   predicate(n->get_int() == 16);
 4404   match(ConI);
 4405 
 4406   op_cost(0);
 4407   format %{ %}
 4408   interface(CONST_INTER);
 4409 %}
 4410 
 4411 operand immI_24()
 4412 %{
 4413   predicate(n->get_int() == 24);
 4414   match(ConI);
 4415 
 4416   op_cost(0);
 4417   format %{ %}
 4418   interface(CONST_INTER);
 4419 %}
 4420 
 4421 operand immI_32()
 4422 %{
 4423   predicate(n->get_int() == 32);
 4424   match(ConI);
 4425 
 4426   op_cost(0);
 4427   format %{ %}
 4428   interface(CONST_INTER);
 4429 %}
 4430 
 4431 operand immI_48()
 4432 %{
 4433   predicate(n->get_int() == 48);
 4434   match(ConI);
 4435 
 4436   op_cost(0);
 4437   format %{ %}
 4438   interface(CONST_INTER);
 4439 %}
 4440 
 4441 operand immI_56()
 4442 %{
 4443   predicate(n->get_int() == 56);
 4444   match(ConI);
 4445 
 4446   op_cost(0);
 4447   format %{ %}
 4448   interface(CONST_INTER);
 4449 %}
 4450 
 4451 operand immI_63()
 4452 %{
 4453   predicate(n->get_int() == 63);
 4454   match(ConI);
 4455 
 4456   op_cost(0);
 4457   format %{ %}
 4458   interface(CONST_INTER);
 4459 %}
 4460 
 4461 operand immI_64()
 4462 %{
 4463   predicate(n->get_int() == 64);
 4464   match(ConI);
 4465 
 4466   op_cost(0);
 4467   format %{ %}
 4468   interface(CONST_INTER);
 4469 %}
 4470 
 4471 operand immI_255()
 4472 %{
 4473   predicate(n->get_int() == 255);
 4474   match(ConI);
 4475 
 4476   op_cost(0);
 4477   format %{ %}
 4478   interface(CONST_INTER);
 4479 %}
 4480 
 4481 operand immI_65535()
 4482 %{
 4483   predicate(n->get_int() == 65535);
 4484   match(ConI);
 4485 
 4486   op_cost(0);
 4487   format %{ %}
 4488   interface(CONST_INTER);
 4489 %}
 4490 
 4491 operand immL_255()
 4492 %{
 4493   predicate(n->get_long() == 255L);
 4494   match(ConL);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 operand immL_65535()
 4502 %{
 4503   predicate(n->get_long() == 65535L);
 4504   match(ConL);
 4505 
 4506   op_cost(0);
 4507   format %{ %}
 4508   interface(CONST_INTER);
 4509 %}
 4510 
 4511 operand immL_4294967295()
 4512 %{
 4513   predicate(n->get_long() == 4294967295L);
 4514   match(ConL);
 4515 
 4516   op_cost(0);
 4517   format %{ %}
 4518   interface(CONST_INTER);
 4519 %}
 4520 
 4521 operand immL_bitmask()
 4522 %{
 4523   predicate((n->get_long() != 0)
 4524             && ((n->get_long() & 0xc000000000000000l) == 0)
 4525             && is_power_of_2(n->get_long() + 1));
 4526   match(ConL);
 4527 
 4528   op_cost(0);
 4529   format %{ %}
 4530   interface(CONST_INTER);
 4531 %}
 4532 
 4533 operand immI_bitmask()
 4534 %{
 4535   predicate((n->get_int() != 0)
 4536             && ((n->get_int() & 0xc0000000) == 0)
 4537             && is_power_of_2(n->get_int() + 1));
 4538   match(ConI);
 4539 
 4540   op_cost(0);
 4541   format %{ %}
 4542   interface(CONST_INTER);
 4543 %}
 4544 
 4545 operand immL_positive_bitmaskI()
 4546 %{
 4547   predicate((n->get_long() != 0)
 4548             && ((julong)n->get_long() < 0x80000000ULL)
 4549             && is_power_of_2(n->get_long() + 1));
 4550   match(ConL);
 4551 
 4552   op_cost(0);
 4553   format %{ %}
 4554   interface(CONST_INTER);
 4555 %}
 4556 
 4557 // Scale values for scaled offset addressing modes (up to long but not quad)
 4558 operand immIScale()
 4559 %{
 4560   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4561   match(ConI);
 4562 
 4563   op_cost(0);
 4564   format %{ %}
 4565   interface(CONST_INTER);
 4566 %}
 4567 
 4568 // 26 bit signed offset -- for pc-relative branches
 4569 operand immI26()
 4570 %{
 4571   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4572   match(ConI);
 4573 
 4574   op_cost(0);
 4575   format %{ %}
 4576   interface(CONST_INTER);
 4577 %}
 4578 
 4579 // 19 bit signed offset -- for pc-relative loads
 4580 operand immI19()
 4581 %{
 4582   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4583   match(ConI);
 4584 
 4585   op_cost(0);
 4586   format %{ %}
 4587   interface(CONST_INTER);
 4588 %}
 4589 
 4590 // 12 bit unsigned offset -- for base plus immediate loads
 4591 operand immIU12()
 4592 %{
 4593   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4594   match(ConI);
 4595 
 4596   op_cost(0);
 4597   format %{ %}
 4598   interface(CONST_INTER);
 4599 %}
 4600 
 4601 operand immLU12()
 4602 %{
 4603   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4604   match(ConL);
 4605 
 4606   op_cost(0);
 4607   format %{ %}
 4608   interface(CONST_INTER);
 4609 %}
 4610 
 4611 // Offset for scaled or unscaled immediate loads and stores
 4612 operand immIOffset()
 4613 %{
 4614   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4615   match(ConI);
 4616 
 4617   op_cost(0);
 4618   format %{ %}
 4619   interface(CONST_INTER);
 4620 %}
 4621 
 4622 operand immIOffset1()
 4623 %{
 4624   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4625   match(ConI);
 4626 
 4627   op_cost(0);
 4628   format %{ %}
 4629   interface(CONST_INTER);
 4630 %}
 4631 
 4632 operand immIOffset2()
 4633 %{
 4634   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4635   match(ConI);
 4636 
 4637   op_cost(0);
 4638   format %{ %}
 4639   interface(CONST_INTER);
 4640 %}
 4641 
 4642 operand immIOffset4()
 4643 %{
 4644   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4645   match(ConI);
 4646 
 4647   op_cost(0);
 4648   format %{ %}
 4649   interface(CONST_INTER);
 4650 %}
 4651 
 4652 operand immIOffset8()
 4653 %{
 4654   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4655   match(ConI);
 4656 
 4657   op_cost(0);
 4658   format %{ %}
 4659   interface(CONST_INTER);
 4660 %}
 4661 
 4662 operand immIOffset16()
 4663 %{
 4664   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4665   match(ConI);
 4666 
 4667   op_cost(0);
 4668   format %{ %}
 4669   interface(CONST_INTER);
 4670 %}
 4671 
 4672 operand immLoffset()
 4673 %{
 4674   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4675   match(ConL);
 4676 
 4677   op_cost(0);
 4678   format %{ %}
 4679   interface(CONST_INTER);
 4680 %}
 4681 
 4682 operand immLoffset1()
 4683 %{
 4684   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4685   match(ConL);
 4686 
 4687   op_cost(0);
 4688   format %{ %}
 4689   interface(CONST_INTER);
 4690 %}
 4691 
 4692 operand immLoffset2()
 4693 %{
 4694   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4695   match(ConL);
 4696 
 4697   op_cost(0);
 4698   format %{ %}
 4699   interface(CONST_INTER);
 4700 %}
 4701 
 4702 operand immLoffset4()
 4703 %{
 4704   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4705   match(ConL);
 4706 
 4707   op_cost(0);
 4708   format %{ %}
 4709   interface(CONST_INTER);
 4710 %}
 4711 
 4712 operand immLoffset8()
 4713 %{
 4714   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4715   match(ConL);
 4716 
 4717   op_cost(0);
 4718   format %{ %}
 4719   interface(CONST_INTER);
 4720 %}
 4721 
 4722 operand immLoffset16()
 4723 %{
 4724   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4725   match(ConL);
 4726 
 4727   op_cost(0);
 4728   format %{ %}
 4729   interface(CONST_INTER);
 4730 %}
 4731 
 4732 // 8 bit signed value.
 4733 operand immI8()
 4734 %{
 4735   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4736   match(ConI);
 4737 
 4738   op_cost(0);
 4739   format %{ %}
 4740   interface(CONST_INTER);
 4741 %}
 4742 
 4743 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4744 operand immI8_shift8()
 4745 %{
 4746   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4747             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4748   match(ConI);
 4749 
 4750   op_cost(0);
 4751   format %{ %}
 4752   interface(CONST_INTER);
 4753 %}
 4754 
 4755 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4756 operand immL8_shift8()
 4757 %{
 4758   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4759             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4760   match(ConL);
 4761 
 4762   op_cost(0);
 4763   format %{ %}
 4764   interface(CONST_INTER);
 4765 %}
 4766 
 4767 // 8 bit integer valid for vector add sub immediate
 4768 operand immBAddSubV()
 4769 %{
 4770   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4771   match(ConI);
 4772 
 4773   op_cost(0);
 4774   format %{ %}
 4775   interface(CONST_INTER);
 4776 %}
 4777 
 4778 // 32 bit integer valid for add sub immediate
 4779 operand immIAddSub()
 4780 %{
 4781   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4782   match(ConI);
 4783   op_cost(0);
 4784   format %{ %}
 4785   interface(CONST_INTER);
 4786 %}
 4787 
 4788 // 32 bit integer valid for vector add sub immediate
 4789 operand immIAddSubV()
 4790 %{
 4791   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4792   match(ConI);
 4793 
 4794   op_cost(0);
 4795   format %{ %}
 4796   interface(CONST_INTER);
 4797 %}
 4798 
 4799 // 32 bit unsigned integer valid for logical immediate
 4800 
 4801 operand immBLog()
 4802 %{
 4803   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4804   match(ConI);
 4805 
 4806   op_cost(0);
 4807   format %{ %}
 4808   interface(CONST_INTER);
 4809 %}
 4810 
 4811 operand immSLog()
 4812 %{
 4813   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4814   match(ConI);
 4815 
 4816   op_cost(0);
 4817   format %{ %}
 4818   interface(CONST_INTER);
 4819 %}
 4820 
 4821 operand immILog()
 4822 %{
 4823   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4824   match(ConI);
 4825 
 4826   op_cost(0);
 4827   format %{ %}
 4828   interface(CONST_INTER);
 4829 %}
 4830 
 4831 // Integer operands 64 bit
 4832 // 64 bit immediate
 4833 operand immL()
 4834 %{
 4835   match(ConL);
 4836 
 4837   op_cost(0);
 4838   format %{ %}
 4839   interface(CONST_INTER);
 4840 %}
 4841 
 4842 // 64 bit zero
 4843 operand immL0()
 4844 %{
 4845   predicate(n->get_long() == 0);
 4846   match(ConL);
 4847 
 4848   op_cost(0);
 4849   format %{ %}
 4850   interface(CONST_INTER);
 4851 %}
 4852 
 4853 // 64 bit unit increment
 4854 operand immL_1()
 4855 %{
 4856   predicate(n->get_long() == 1);
 4857   match(ConL);
 4858 
 4859   op_cost(0);
 4860   format %{ %}
 4861   interface(CONST_INTER);
 4862 %}
 4863 
 4864 // 64 bit unit decrement
 4865 operand immL_M1()
 4866 %{
 4867   predicate(n->get_long() == -1);
 4868   match(ConL);
 4869 
 4870   op_cost(0);
 4871   format %{ %}
 4872   interface(CONST_INTER);
 4873 %}
 4874 
 4875 // 32 bit offset of pc in thread anchor
 4876 
 4877 operand immL_pc_off()
 4878 %{
 4879   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4880                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4881   match(ConL);
 4882 
 4883   op_cost(0);
 4884   format %{ %}
 4885   interface(CONST_INTER);
 4886 %}
 4887 
 4888 // 64 bit integer valid for add sub immediate
 4889 operand immLAddSub()
 4890 %{
 4891   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4892   match(ConL);
 4893   op_cost(0);
 4894   format %{ %}
 4895   interface(CONST_INTER);
 4896 %}
 4897 
 4898 // 64 bit integer valid for addv subv immediate
 4899 operand immLAddSubV()
 4900 %{
 4901   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4902   match(ConL);
 4903 
 4904   op_cost(0);
 4905   format %{ %}
 4906   interface(CONST_INTER);
 4907 %}
 4908 
 4909 // 64 bit integer valid for logical immediate
 4910 operand immLLog()
 4911 %{
 4912   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4913   match(ConL);
 4914   op_cost(0);
 4915   format %{ %}
 4916   interface(CONST_INTER);
 4917 %}
 4918 
 4919 // Long Immediate: low 32-bit mask
 4920 operand immL_32bits()
 4921 %{
 4922   predicate(n->get_long() == 0xFFFFFFFFL);
 4923   match(ConL);
 4924   op_cost(0);
 4925   format %{ %}
 4926   interface(CONST_INTER);
 4927 %}
 4928 
 4929 // Pointer operands
 4930 // Pointer Immediate
 4931 operand immP()
 4932 %{
 4933   match(ConP);
 4934 
 4935   op_cost(0);
 4936   format %{ %}
 4937   interface(CONST_INTER);
 4938 %}
 4939 
 4940 // NULL Pointer Immediate
 4941 operand immP0()
 4942 %{
 4943   predicate(n->get_ptr() == 0);
 4944   match(ConP);
 4945 
 4946   op_cost(0);
 4947   format %{ %}
 4948   interface(CONST_INTER);
 4949 %}
 4950 
 4951 // Pointer Immediate One
 4952 // this is used in object initialization (initial object header)
 4953 operand immP_1()
 4954 %{
 4955   predicate(n->get_ptr() == 1);
 4956   match(ConP);
 4957 
 4958   op_cost(0);
 4959   format %{ %}
 4960   interface(CONST_INTER);
 4961 %}
 4962 
 4963 // Card Table Byte Map Base
 4964 operand immByteMapBase()
 4965 %{
 4966   // Get base of card map
 4967   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4968             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4969   match(ConP);
 4970 
 4971   op_cost(0);
 4972   format %{ %}
 4973   interface(CONST_INTER);
 4974 %}
 4975 
 4976 // Pointer Immediate Minus One
 4977 // this is used when we want to write the current PC to the thread anchor
 4978 operand immP_M1()
 4979 %{
 4980   predicate(n->get_ptr() == -1);
 4981   match(ConP);
 4982 
 4983   op_cost(0);
 4984   format %{ %}
 4985   interface(CONST_INTER);
 4986 %}
 4987 
 4988 // Pointer Immediate Minus Two
 4989 // this is used when we want to write the current PC to the thread anchor
 4990 operand immP_M2()
 4991 %{
 4992   predicate(n->get_ptr() == -2);
 4993   match(ConP);
 4994 
 4995   op_cost(0);
 4996   format %{ %}
 4997   interface(CONST_INTER);
 4998 %}
 4999 
 5000 // Float and Double operands
 5001 // Double Immediate
 5002 operand immD()
 5003 %{
 5004   match(ConD);
 5005   op_cost(0);
 5006   format %{ %}
 5007   interface(CONST_INTER);
 5008 %}
 5009 
 5010 // Double Immediate: +0.0d
 5011 operand immD0()
 5012 %{
 5013   predicate(jlong_cast(n->getd()) == 0);
 5014   match(ConD);
 5015 
 5016   op_cost(0);
 5017   format %{ %}
 5018   interface(CONST_INTER);
 5019 %}
 5020 
 5021 // constant 'double +0.0'.
 5022 operand immDPacked()
 5023 %{
 5024   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 5025   match(ConD);
 5026   op_cost(0);
 5027   format %{ %}
 5028   interface(CONST_INTER);
 5029 %}
 5030 
 5031 // Float Immediate
 5032 operand immF()
 5033 %{
 5034   match(ConF);
 5035   op_cost(0);
 5036   format %{ %}
 5037   interface(CONST_INTER);
 5038 %}
 5039 
 5040 // Float Immediate: +0.0f.
 5041 operand immF0()
 5042 %{
 5043   predicate(jint_cast(n->getf()) == 0);
 5044   match(ConF);
 5045 
 5046   op_cost(0);
 5047   format %{ %}
 5048   interface(CONST_INTER);
 5049 %}
 5050 
 5051 //
 5052 operand immFPacked()
 5053 %{
 5054   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 5055   match(ConF);
 5056   op_cost(0);
 5057   format %{ %}
 5058   interface(CONST_INTER);
 5059 %}
 5060 
 5061 // Narrow pointer operands
 5062 // Narrow Pointer Immediate
 5063 operand immN()
 5064 %{
 5065   match(ConN);
 5066 
 5067   op_cost(0);
 5068   format %{ %}
 5069   interface(CONST_INTER);
 5070 %}
 5071 
 5072 // Narrow NULL Pointer Immediate
 5073 operand immN0()
 5074 %{
 5075   predicate(n->get_narrowcon() == 0);
 5076   match(ConN);
 5077 
 5078   op_cost(0);
 5079   format %{ %}
 5080   interface(CONST_INTER);
 5081 %}
 5082 
 5083 operand immNKlass()
 5084 %{
 5085   match(ConNKlass);
 5086 
 5087   op_cost(0);
 5088   format %{ %}
 5089   interface(CONST_INTER);
 5090 %}
 5091 
 5092 // Integer 32 bit Register Operands
 5093 // Integer 32 bitRegister (excludes SP)
 5094 operand iRegI()
 5095 %{
 5096   constraint(ALLOC_IN_RC(any_reg32));
 5097   match(RegI);
 5098   match(iRegINoSp);
 5099   op_cost(0);
 5100   format %{ %}
 5101   interface(REG_INTER);
 5102 %}
 5103 
 5104 // Integer 32 bit Register not Special
 5105 operand iRegINoSp()
 5106 %{
 5107   constraint(ALLOC_IN_RC(no_special_reg32));
 5108   match(RegI);
 5109   op_cost(0);
 5110   format %{ %}
 5111   interface(REG_INTER);
 5112 %}
 5113 
 5114 // Integer 64 bit Register Operands
 5115 // Integer 64 bit Register (includes SP)
 5116 operand iRegL()
 5117 %{
 5118   constraint(ALLOC_IN_RC(any_reg));
 5119   match(RegL);
 5120   match(iRegLNoSp);
 5121   op_cost(0);
 5122   format %{ %}
 5123   interface(REG_INTER);
 5124 %}
 5125 
 5126 // Integer 64 bit Register not Special
 5127 operand iRegLNoSp()
 5128 %{
 5129   constraint(ALLOC_IN_RC(no_special_reg));
 5130   match(RegL);
 5131   match(iRegL_R0);
 5132   format %{ %}
 5133   interface(REG_INTER);
 5134 %}
 5135 
 5136 // Pointer Register Operands
 5137 // Pointer Register
 5138 operand iRegP()
 5139 %{
 5140   constraint(ALLOC_IN_RC(ptr_reg));
 5141   match(RegP);
 5142   match(iRegPNoSp);
 5143   match(iRegP_R0);
 5144   //match(iRegP_R2);
 5145   //match(iRegP_R4);
 5146   //match(iRegP_R5);
 5147   match(thread_RegP);
 5148   op_cost(0);
 5149   format %{ %}
 5150   interface(REG_INTER);
 5151 %}
 5152 
 5153 // Pointer 64 bit Register not Special
 5154 operand iRegPNoSp()
 5155 %{
 5156   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 5157   match(RegP);
 5158   // match(iRegP);
 5159   // match(iRegP_R0);
 5160   // match(iRegP_R2);
 5161   // match(iRegP_R4);
 5162   // match(iRegP_R5);
 5163   // match(thread_RegP);
 5164   op_cost(0);
 5165   format %{ %}
 5166   interface(REG_INTER);
 5167 %}
 5168 
 5169 // Pointer 64 bit Register R0 only
 5170 operand iRegP_R0()
 5171 %{
 5172   constraint(ALLOC_IN_RC(r0_reg));
 5173   match(RegP);
 5174   // match(iRegP);
 5175   match(iRegPNoSp);
 5176   op_cost(0);
 5177   format %{ %}
 5178   interface(REG_INTER);
 5179 %}
 5180 
 5181 // Pointer 64 bit Register R1 only
 5182 operand iRegP_R1()
 5183 %{
 5184   constraint(ALLOC_IN_RC(r1_reg));
 5185   match(RegP);
 5186   // match(iRegP);
 5187   match(iRegPNoSp);
 5188   op_cost(0);
 5189   format %{ %}
 5190   interface(REG_INTER);
 5191 %}
 5192 
 5193 // Pointer 64 bit Register R2 only
 5194 operand iRegP_R2()
 5195 %{
 5196   constraint(ALLOC_IN_RC(r2_reg));
 5197   match(RegP);
 5198   // match(iRegP);
 5199   match(iRegPNoSp);
 5200   op_cost(0);
 5201   format %{ %}
 5202   interface(REG_INTER);
 5203 %}
 5204 
 5205 // Pointer 64 bit Register R3 only
 5206 operand iRegP_R3()
 5207 %{
 5208   constraint(ALLOC_IN_RC(r3_reg));
 5209   match(RegP);
 5210   // match(iRegP);
 5211   match(iRegPNoSp);
 5212   op_cost(0);
 5213   format %{ %}
 5214   interface(REG_INTER);
 5215 %}
 5216 
 5217 // Pointer 64 bit Register R4 only
 5218 operand iRegP_R4()
 5219 %{
 5220   constraint(ALLOC_IN_RC(r4_reg));
 5221   match(RegP);
 5222   // match(iRegP);
 5223   match(iRegPNoSp);
 5224   op_cost(0);
 5225   format %{ %}
 5226   interface(REG_INTER);
 5227 %}
 5228 
 5229 // Pointer 64 bit Register R5 only
 5230 operand iRegP_R5()
 5231 %{
 5232   constraint(ALLOC_IN_RC(r5_reg));
 5233   match(RegP);
 5234   // match(iRegP);
 5235   match(iRegPNoSp);
 5236   op_cost(0);
 5237   format %{ %}
 5238   interface(REG_INTER);
 5239 %}
 5240 
 5241 // Pointer 64 bit Register R10 only
 5242 operand iRegP_R10()
 5243 %{
 5244   constraint(ALLOC_IN_RC(r10_reg));
 5245   match(RegP);
 5246   // match(iRegP);
 5247   match(iRegPNoSp);
 5248   op_cost(0);
 5249   format %{ %}
 5250   interface(REG_INTER);
 5251 %}
 5252 
 5253 // Long 64 bit Register R0 only
 5254 operand iRegL_R0()
 5255 %{
 5256   constraint(ALLOC_IN_RC(r0_reg));
 5257   match(RegL);
 5258   match(iRegLNoSp);
 5259   op_cost(0);
 5260   format %{ %}
 5261   interface(REG_INTER);
 5262 %}
 5263 
 5264 // Long 64 bit Register R2 only
 5265 operand iRegL_R2()
 5266 %{
 5267   constraint(ALLOC_IN_RC(r2_reg));
 5268   match(RegL);
 5269   match(iRegLNoSp);
 5270   op_cost(0);
 5271   format %{ %}
 5272   interface(REG_INTER);
 5273 %}
 5274 
 5275 // Long 64 bit Register R3 only
 5276 operand iRegL_R3()
 5277 %{
 5278   constraint(ALLOC_IN_RC(r3_reg));
 5279   match(RegL);
 5280   match(iRegLNoSp);
 5281   op_cost(0);
 5282   format %{ %}
 5283   interface(REG_INTER);
 5284 %}
 5285 
 5286 // Long 64 bit Register R11 only
 5287 operand iRegL_R11()
 5288 %{
 5289   constraint(ALLOC_IN_RC(r11_reg));
 5290   match(RegL);
 5291   match(iRegLNoSp);
 5292   op_cost(0);
 5293   format %{ %}
 5294   interface(REG_INTER);
 5295 %}
 5296 
 5297 // Pointer 64 bit Register FP only
 5298 operand iRegP_FP()
 5299 %{
 5300   constraint(ALLOC_IN_RC(fp_reg));
 5301   match(RegP);
 5302   // match(iRegP);
 5303   op_cost(0);
 5304   format %{ %}
 5305   interface(REG_INTER);
 5306 %}
 5307 
 5308 // Register R0 only
 5309 operand iRegI_R0()
 5310 %{
 5311   constraint(ALLOC_IN_RC(int_r0_reg));
 5312   match(RegI);
 5313   match(iRegINoSp);
 5314   op_cost(0);
 5315   format %{ %}
 5316   interface(REG_INTER);
 5317 %}
 5318 
 5319 // Register R2 only
 5320 operand iRegI_R2()
 5321 %{
 5322   constraint(ALLOC_IN_RC(int_r2_reg));
 5323   match(RegI);
 5324   match(iRegINoSp);
 5325   op_cost(0);
 5326   format %{ %}
 5327   interface(REG_INTER);
 5328 %}
 5329 
 5330 // Register R3 only
 5331 operand iRegI_R3()
 5332 %{
 5333   constraint(ALLOC_IN_RC(int_r3_reg));
 5334   match(RegI);
 5335   match(iRegINoSp);
 5336   op_cost(0);
 5337   format %{ %}
 5338   interface(REG_INTER);
 5339 %}
 5340 
 5341 
 5342 // Register R4 only
 5343 operand iRegI_R4()
 5344 %{
 5345   constraint(ALLOC_IN_RC(int_r4_reg));
 5346   match(RegI);
 5347   match(iRegINoSp);
 5348   op_cost(0);
 5349   format %{ %}
 5350   interface(REG_INTER);
 5351 %}
 5352 
 5353 
 5354 // Pointer Register Operands
 5355 // Narrow Pointer Register
 5356 operand iRegN()
 5357 %{
 5358   constraint(ALLOC_IN_RC(any_reg32));
 5359   match(RegN);
 5360   match(iRegNNoSp);
 5361   op_cost(0);
 5362   format %{ %}
 5363   interface(REG_INTER);
 5364 %}
 5365 
 5366 operand iRegN_R0()
 5367 %{
 5368   constraint(ALLOC_IN_RC(r0_reg));
 5369   match(iRegN);
 5370   op_cost(0);
 5371   format %{ %}
 5372   interface(REG_INTER);
 5373 %}
 5374 
 5375 operand iRegN_R2()
 5376 %{
 5377   constraint(ALLOC_IN_RC(r2_reg));
 5378   match(iRegN);
 5379   op_cost(0);
 5380   format %{ %}
 5381   interface(REG_INTER);
 5382 %}
 5383 
 5384 operand iRegN_R3()
 5385 %{
 5386   constraint(ALLOC_IN_RC(r3_reg));
 5387   match(iRegN);
 5388   op_cost(0);
 5389   format %{ %}
 5390   interface(REG_INTER);
 5391 %}
 5392 
 5393 // Integer 64 bit Register not Special
 5394 operand iRegNNoSp()
 5395 %{
 5396   constraint(ALLOC_IN_RC(no_special_reg32));
 5397   match(RegN);
 5398   op_cost(0);
 5399   format %{ %}
 5400   interface(REG_INTER);
 5401 %}
 5402 
 5403 // heap base register -- used for encoding immN0
 5404 
 5405 operand iRegIHeapbase()
 5406 %{
 5407   constraint(ALLOC_IN_RC(heapbase_reg));
 5408   match(RegI);
 5409   op_cost(0);
 5410   format %{ %}
 5411   interface(REG_INTER);
 5412 %}
 5413 
 5414 // Float Register
 5415 // Float register operands
 5416 operand vRegF()
 5417 %{
 5418   constraint(ALLOC_IN_RC(float_reg));
 5419   match(RegF);
 5420 
 5421   op_cost(0);
 5422   format %{ %}
 5423   interface(REG_INTER);
 5424 %}
 5425 
 5426 // Double Register
 5427 // Double register operands
 5428 operand vRegD()
 5429 %{
 5430   constraint(ALLOC_IN_RC(double_reg));
 5431   match(RegD);
 5432 
 5433   op_cost(0);
 5434   format %{ %}
 5435   interface(REG_INTER);
 5436 %}
 5437 
 5438 // Generic vector class. This will be used for
 5439 // all vector operands, including NEON and SVE,
 5440 // but currently only used for SVE VecA.
 5441 operand vReg()
 5442 %{
 5443   constraint(ALLOC_IN_RC(vectora_reg));
 5444   match(VecA);
 5445   op_cost(0);
 5446   format %{ %}
 5447   interface(REG_INTER);
 5448 %}
 5449 
 5450 operand vecD()
 5451 %{
 5452   constraint(ALLOC_IN_RC(vectord_reg));
 5453   match(VecD);
 5454 
 5455   op_cost(0);
 5456   format %{ %}
 5457   interface(REG_INTER);
 5458 %}
 5459 
 5460 operand vecX()
 5461 %{
 5462   constraint(ALLOC_IN_RC(vectorx_reg));
 5463   match(VecX);
 5464 
 5465   op_cost(0);
 5466   format %{ %}
 5467   interface(REG_INTER);
 5468 %}
 5469 
 5470 operand vRegD_V0()
 5471 %{
 5472   constraint(ALLOC_IN_RC(v0_reg));
 5473   match(RegD);
 5474   op_cost(0);
 5475   format %{ %}
 5476   interface(REG_INTER);
 5477 %}
 5478 
 5479 operand vRegD_V1()
 5480 %{
 5481   constraint(ALLOC_IN_RC(v1_reg));
 5482   match(RegD);
 5483   op_cost(0);
 5484   format %{ %}
 5485   interface(REG_INTER);
 5486 %}
 5487 
 5488 operand vRegD_V2()
 5489 %{
 5490   constraint(ALLOC_IN_RC(v2_reg));
 5491   match(RegD);
 5492   op_cost(0);
 5493   format %{ %}
 5494   interface(REG_INTER);
 5495 %}
 5496 
 5497 operand vRegD_V3()
 5498 %{
 5499   constraint(ALLOC_IN_RC(v3_reg));
 5500   match(RegD);
 5501   op_cost(0);
 5502   format %{ %}
 5503   interface(REG_INTER);
 5504 %}
 5505 
 5506 operand vRegD_V4()
 5507 %{
 5508   constraint(ALLOC_IN_RC(v4_reg));
 5509   match(RegD);
 5510   op_cost(0);
 5511   format %{ %}
 5512   interface(REG_INTER);
 5513 %}
 5514 
 5515 operand vRegD_V5()
 5516 %{
 5517   constraint(ALLOC_IN_RC(v5_reg));
 5518   match(RegD);
 5519   op_cost(0);
 5520   format %{ %}
 5521   interface(REG_INTER);
 5522 %}
 5523 
 5524 operand vRegD_V6()
 5525 %{
 5526   constraint(ALLOC_IN_RC(v6_reg));
 5527   match(RegD);
 5528   op_cost(0);
 5529   format %{ %}
 5530   interface(REG_INTER);
 5531 %}
 5532 
 5533 operand vRegD_V7()
 5534 %{
 5535   constraint(ALLOC_IN_RC(v7_reg));
 5536   match(RegD);
 5537   op_cost(0);
 5538   format %{ %}
 5539   interface(REG_INTER);
 5540 %}
 5541 
 5542 operand vRegD_V8()
 5543 %{
 5544   constraint(ALLOC_IN_RC(v8_reg));
 5545   match(RegD);
 5546   op_cost(0);
 5547   format %{ %}
 5548   interface(REG_INTER);
 5549 %}
 5550 
 5551 operand vRegD_V9()
 5552 %{
 5553   constraint(ALLOC_IN_RC(v9_reg));
 5554   match(RegD);
 5555   op_cost(0);
 5556   format %{ %}
 5557   interface(REG_INTER);
 5558 %}
 5559 
 5560 operand vRegD_V10()
 5561 %{
 5562   constraint(ALLOC_IN_RC(v10_reg));
 5563   match(RegD);
 5564   op_cost(0);
 5565   format %{ %}
 5566   interface(REG_INTER);
 5567 %}
 5568 
 5569 operand vRegD_V11()
 5570 %{
 5571   constraint(ALLOC_IN_RC(v11_reg));
 5572   match(RegD);
 5573   op_cost(0);
 5574   format %{ %}
 5575   interface(REG_INTER);
 5576 %}
 5577 
 5578 operand vRegD_V12()
 5579 %{
 5580   constraint(ALLOC_IN_RC(v12_reg));
 5581   match(RegD);
 5582   op_cost(0);
 5583   format %{ %}
 5584   interface(REG_INTER);
 5585 %}
 5586 
 5587 operand vRegD_V13()
 5588 %{
 5589   constraint(ALLOC_IN_RC(v13_reg));
 5590   match(RegD);
 5591   op_cost(0);
 5592   format %{ %}
 5593   interface(REG_INTER);
 5594 %}
 5595 
 5596 operand vRegD_V14()
 5597 %{
 5598   constraint(ALLOC_IN_RC(v14_reg));
 5599   match(RegD);
 5600   op_cost(0);
 5601   format %{ %}
 5602   interface(REG_INTER);
 5603 %}
 5604 
 5605 operand vRegD_V15()
 5606 %{
 5607   constraint(ALLOC_IN_RC(v15_reg));
 5608   match(RegD);
 5609   op_cost(0);
 5610   format %{ %}
 5611   interface(REG_INTER);
 5612 %}
 5613 
 5614 operand vRegD_V16()
 5615 %{
 5616   constraint(ALLOC_IN_RC(v16_reg));
 5617   match(RegD);
 5618   op_cost(0);
 5619   format %{ %}
 5620   interface(REG_INTER);
 5621 %}
 5622 
 5623 operand vRegD_V17()
 5624 %{
 5625   constraint(ALLOC_IN_RC(v17_reg));
 5626   match(RegD);
 5627   op_cost(0);
 5628   format %{ %}
 5629   interface(REG_INTER);
 5630 %}
 5631 
 5632 operand vRegD_V18()
 5633 %{
 5634   constraint(ALLOC_IN_RC(v18_reg));
 5635   match(RegD);
 5636   op_cost(0);
 5637   format %{ %}
 5638   interface(REG_INTER);
 5639 %}
 5640 
 5641 operand vRegD_V19()
 5642 %{
 5643   constraint(ALLOC_IN_RC(v19_reg));
 5644   match(RegD);
 5645   op_cost(0);
 5646   format %{ %}
 5647   interface(REG_INTER);
 5648 %}
 5649 
 5650 operand vRegD_V20()
 5651 %{
 5652   constraint(ALLOC_IN_RC(v20_reg));
 5653   match(RegD);
 5654   op_cost(0);
 5655   format %{ %}
 5656   interface(REG_INTER);
 5657 %}
 5658 
 5659 operand vRegD_V21()
 5660 %{
 5661   constraint(ALLOC_IN_RC(v21_reg));
 5662   match(RegD);
 5663   op_cost(0);
 5664   format %{ %}
 5665   interface(REG_INTER);
 5666 %}
 5667 
 5668 operand vRegD_V22()
 5669 %{
 5670   constraint(ALLOC_IN_RC(v22_reg));
 5671   match(RegD);
 5672   op_cost(0);
 5673   format %{ %}
 5674   interface(REG_INTER);
 5675 %}
 5676 
 5677 operand vRegD_V23()
 5678 %{
 5679   constraint(ALLOC_IN_RC(v23_reg));
 5680   match(RegD);
 5681   op_cost(0);
 5682   format %{ %}
 5683   interface(REG_INTER);
 5684 %}
 5685 
 5686 operand vRegD_V24()
 5687 %{
 5688   constraint(ALLOC_IN_RC(v24_reg));
 5689   match(RegD);
 5690   op_cost(0);
 5691   format %{ %}
 5692   interface(REG_INTER);
 5693 %}
 5694 
 5695 operand vRegD_V25()
 5696 %{
 5697   constraint(ALLOC_IN_RC(v25_reg));
 5698   match(RegD);
 5699   op_cost(0);
 5700   format %{ %}
 5701   interface(REG_INTER);
 5702 %}
 5703 
 5704 operand vRegD_V26()
 5705 %{
 5706   constraint(ALLOC_IN_RC(v26_reg));
 5707   match(RegD);
 5708   op_cost(0);
 5709   format %{ %}
 5710   interface(REG_INTER);
 5711 %}
 5712 
 5713 operand vRegD_V27()
 5714 %{
 5715   constraint(ALLOC_IN_RC(v27_reg));
 5716   match(RegD);
 5717   op_cost(0);
 5718   format %{ %}
 5719   interface(REG_INTER);
 5720 %}
 5721 
 5722 operand vRegD_V28()
 5723 %{
 5724   constraint(ALLOC_IN_RC(v28_reg));
 5725   match(RegD);
 5726   op_cost(0);
 5727   format %{ %}
 5728   interface(REG_INTER);
 5729 %}
 5730 
 5731 operand vRegD_V29()
 5732 %{
 5733   constraint(ALLOC_IN_RC(v29_reg));
 5734   match(RegD);
 5735   op_cost(0);
 5736   format %{ %}
 5737   interface(REG_INTER);
 5738 %}
 5739 
 5740 operand vRegD_V30()
 5741 %{
 5742   constraint(ALLOC_IN_RC(v30_reg));
 5743   match(RegD);
 5744   op_cost(0);
 5745   format %{ %}
 5746   interface(REG_INTER);
 5747 %}
 5748 
 5749 operand vRegD_V31()
 5750 %{
 5751   constraint(ALLOC_IN_RC(v31_reg));
 5752   match(RegD);
 5753   op_cost(0);
 5754   format %{ %}
 5755   interface(REG_INTER);
 5756 %}
 5757 
 5758 operand pReg()
 5759 %{
 5760   constraint(ALLOC_IN_RC(pr_reg));
 5761   match(RegVectMask);
 5762   match(pRegGov);
 5763   op_cost(0);
 5764   format %{ %}
 5765   interface(REG_INTER);
 5766 %}
 5767 
 5768 operand pRegGov()
 5769 %{
 5770   constraint(ALLOC_IN_RC(gov_pr));
 5771   match(RegVectMask);
 5772   op_cost(0);
 5773   format %{ %}
 5774   interface(REG_INTER);
 5775 %}
 5776 
 5777 // Flags register, used as output of signed compare instructions
 5778 
 5779 // note that on AArch64 we also use this register as the output for
 5780 // for floating point compare instructions (CmpF CmpD). this ensures
 5781 // that ordered inequality tests use GT, GE, LT or LE none of which
 5782 // pass through cases where the result is unordered i.e. one or both
 5783 // inputs to the compare is a NaN. this means that the ideal code can
 5784 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5785 // (where the comparison should always fail). EQ and NE tests are
 5786 // always generated in ideal code so that unordered folds into the NE
 5787 // case, matching the behaviour of AArch64 NE.
 5788 //
 5789 // This differs from x86 where the outputs of FP compares use a
 5790 // special FP flags registers and where compares based on this
 5791 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5792 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5793 // to explicitly handle the unordered case in branches. x86 also has
 5794 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5795 
 5796 operand rFlagsReg()
 5797 %{
 5798   constraint(ALLOC_IN_RC(int_flags));
 5799   match(RegFlags);
 5800 
 5801   op_cost(0);
 5802   format %{ "RFLAGS" %}
 5803   interface(REG_INTER);
 5804 %}
 5805 
 5806 // Flags register, used as output of unsigned compare instructions
 5807 operand rFlagsRegU()
 5808 %{
 5809   constraint(ALLOC_IN_RC(int_flags));
 5810   match(RegFlags);
 5811 
 5812   op_cost(0);
 5813   format %{ "RFLAGSU" %}
 5814   interface(REG_INTER);
 5815 %}
 5816 
 5817 // Special Registers
 5818 
 5819 // Method Register
 5820 operand inline_cache_RegP(iRegP reg)
 5821 %{
 5822   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5823   match(reg);
 5824   match(iRegPNoSp);
 5825   op_cost(0);
 5826   format %{ %}
 5827   interface(REG_INTER);
 5828 %}
 5829 
 5830 // Thread Register
 5831 operand thread_RegP(iRegP reg)
 5832 %{
 5833   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5834   match(reg);
 5835   op_cost(0);
 5836   format %{ %}
 5837   interface(REG_INTER);
 5838 %}
 5839 
 5840 operand lr_RegP(iRegP reg)
 5841 %{
 5842   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5843   match(reg);
 5844   op_cost(0);
 5845   format %{ %}
 5846   interface(REG_INTER);
 5847 %}
 5848 
 5849 //----------Memory Operands----------------------------------------------------
 5850 
 5851 operand indirect(iRegP reg)
 5852 %{
 5853   constraint(ALLOC_IN_RC(ptr_reg));
 5854   match(reg);
 5855   op_cost(0);
 5856   format %{ "[$reg]" %}
 5857   interface(MEMORY_INTER) %{
 5858     base($reg);
 5859     index(0xffffffff);
 5860     scale(0x0);
 5861     disp(0x0);
 5862   %}
 5863 %}
 5864 
 5865 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5866 %{
 5867   constraint(ALLOC_IN_RC(ptr_reg));
 5868   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5869   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5870   op_cost(0);
 5871   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5872   interface(MEMORY_INTER) %{
 5873     base($reg);
 5874     index($ireg);
 5875     scale($scale);
 5876     disp(0x0);
 5877   %}
 5878 %}
 5879 
 5880 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5881 %{
 5882   constraint(ALLOC_IN_RC(ptr_reg));
 5883   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5884   match(AddP reg (LShiftL lreg scale));
 5885   op_cost(0);
 5886   format %{ "$reg, $lreg lsl($scale)" %}
 5887   interface(MEMORY_INTER) %{
 5888     base($reg);
 5889     index($lreg);
 5890     scale($scale);
 5891     disp(0x0);
 5892   %}
 5893 %}
 5894 
 5895 operand indIndexI2L(iRegP reg, iRegI ireg)
 5896 %{
 5897   constraint(ALLOC_IN_RC(ptr_reg));
 5898   match(AddP reg (ConvI2L ireg));
 5899   op_cost(0);
 5900   format %{ "$reg, $ireg, 0, I2L" %}
 5901   interface(MEMORY_INTER) %{
 5902     base($reg);
 5903     index($ireg);
 5904     scale(0x0);
 5905     disp(0x0);
 5906   %}
 5907 %}
 5908 
 5909 operand indIndex(iRegP reg, iRegL lreg)
 5910 %{
 5911   constraint(ALLOC_IN_RC(ptr_reg));
 5912   match(AddP reg lreg);
 5913   op_cost(0);
 5914   format %{ "$reg, $lreg" %}
 5915   interface(MEMORY_INTER) %{
 5916     base($reg);
 5917     index($lreg);
 5918     scale(0x0);
 5919     disp(0x0);
 5920   %}
 5921 %}
 5922 
 5923 operand indOffI(iRegP reg, immIOffset off)
 5924 %{
 5925   constraint(ALLOC_IN_RC(ptr_reg));
 5926   match(AddP reg off);
 5927   op_cost(0);
 5928   format %{ "[$reg, $off]" %}
 5929   interface(MEMORY_INTER) %{
 5930     base($reg);
 5931     index(0xffffffff);
 5932     scale(0x0);
 5933     disp($off);
 5934   %}
 5935 %}
 5936 
 5937 operand indOffI1(iRegP reg, immIOffset1 off)
 5938 %{
 5939   constraint(ALLOC_IN_RC(ptr_reg));
 5940   match(AddP reg off);
 5941   op_cost(0);
 5942   format %{ "[$reg, $off]" %}
 5943   interface(MEMORY_INTER) %{
 5944     base($reg);
 5945     index(0xffffffff);
 5946     scale(0x0);
 5947     disp($off);
 5948   %}
 5949 %}
 5950 
 5951 operand indOffI2(iRegP reg, immIOffset2 off)
 5952 %{
 5953   constraint(ALLOC_IN_RC(ptr_reg));
 5954   match(AddP reg off);
 5955   op_cost(0);
 5956   format %{ "[$reg, $off]" %}
 5957   interface(MEMORY_INTER) %{
 5958     base($reg);
 5959     index(0xffffffff);
 5960     scale(0x0);
 5961     disp($off);
 5962   %}
 5963 %}
 5964 
 5965 operand indOffI4(iRegP reg, immIOffset4 off)
 5966 %{
 5967   constraint(ALLOC_IN_RC(ptr_reg));
 5968   match(AddP reg off);
 5969   op_cost(0);
 5970   format %{ "[$reg, $off]" %}
 5971   interface(MEMORY_INTER) %{
 5972     base($reg);
 5973     index(0xffffffff);
 5974     scale(0x0);
 5975     disp($off);
 5976   %}
 5977 %}
 5978 
 5979 operand indOffI8(iRegP reg, immIOffset8 off)
 5980 %{
 5981   constraint(ALLOC_IN_RC(ptr_reg));
 5982   match(AddP reg off);
 5983   op_cost(0);
 5984   format %{ "[$reg, $off]" %}
 5985   interface(MEMORY_INTER) %{
 5986     base($reg);
 5987     index(0xffffffff);
 5988     scale(0x0);
 5989     disp($off);
 5990   %}
 5991 %}
 5992 
 5993 operand indOffI16(iRegP reg, immIOffset16 off)
 5994 %{
 5995   constraint(ALLOC_IN_RC(ptr_reg));
 5996   match(AddP reg off);
 5997   op_cost(0);
 5998   format %{ "[$reg, $off]" %}
 5999   interface(MEMORY_INTER) %{
 6000     base($reg);
 6001     index(0xffffffff);
 6002     scale(0x0);
 6003     disp($off);
 6004   %}
 6005 %}
 6006 
 6007 operand indOffL(iRegP reg, immLoffset off)
 6008 %{
 6009   constraint(ALLOC_IN_RC(ptr_reg));
 6010   match(AddP reg off);
 6011   op_cost(0);
 6012   format %{ "[$reg, $off]" %}
 6013   interface(MEMORY_INTER) %{
 6014     base($reg);
 6015     index(0xffffffff);
 6016     scale(0x0);
 6017     disp($off);
 6018   %}
 6019 %}
 6020 
 6021 operand indOffL1(iRegP reg, immLoffset1 off)
 6022 %{
 6023   constraint(ALLOC_IN_RC(ptr_reg));
 6024   match(AddP reg off);
 6025   op_cost(0);
 6026   format %{ "[$reg, $off]" %}
 6027   interface(MEMORY_INTER) %{
 6028     base($reg);
 6029     index(0xffffffff);
 6030     scale(0x0);
 6031     disp($off);
 6032   %}
 6033 %}
 6034 
 6035 operand indOffL2(iRegP reg, immLoffset2 off)
 6036 %{
 6037   constraint(ALLOC_IN_RC(ptr_reg));
 6038   match(AddP reg off);
 6039   op_cost(0);
 6040   format %{ "[$reg, $off]" %}
 6041   interface(MEMORY_INTER) %{
 6042     base($reg);
 6043     index(0xffffffff);
 6044     scale(0x0);
 6045     disp($off);
 6046   %}
 6047 %}
 6048 
 6049 operand indOffL4(iRegP reg, immLoffset4 off)
 6050 %{
 6051   constraint(ALLOC_IN_RC(ptr_reg));
 6052   match(AddP reg off);
 6053   op_cost(0);
 6054   format %{ "[$reg, $off]" %}
 6055   interface(MEMORY_INTER) %{
 6056     base($reg);
 6057     index(0xffffffff);
 6058     scale(0x0);
 6059     disp($off);
 6060   %}
 6061 %}
 6062 
 6063 operand indOffL8(iRegP reg, immLoffset8 off)
 6064 %{
 6065   constraint(ALLOC_IN_RC(ptr_reg));
 6066   match(AddP reg off);
 6067   op_cost(0);
 6068   format %{ "[$reg, $off]" %}
 6069   interface(MEMORY_INTER) %{
 6070     base($reg);
 6071     index(0xffffffff);
 6072     scale(0x0);
 6073     disp($off);
 6074   %}
 6075 %}
 6076 
 6077 operand indOffL16(iRegP reg, immLoffset16 off)
 6078 %{
 6079   constraint(ALLOC_IN_RC(ptr_reg));
 6080   match(AddP reg off);
 6081   op_cost(0);
 6082   format %{ "[$reg, $off]" %}
 6083   interface(MEMORY_INTER) %{
 6084     base($reg);
 6085     index(0xffffffff);
 6086     scale(0x0);
 6087     disp($off);
 6088   %}
 6089 %}
 6090 
 6091 operand indirectN(iRegN reg)
 6092 %{
 6093   predicate(CompressedOops::shift() == 0);
 6094   constraint(ALLOC_IN_RC(ptr_reg));
 6095   match(DecodeN reg);
 6096   op_cost(0);
 6097   format %{ "[$reg]\t# narrow" %}
 6098   interface(MEMORY_INTER) %{
 6099     base($reg);
 6100     index(0xffffffff);
 6101     scale(0x0);
 6102     disp(0x0);
 6103   %}
 6104 %}
 6105 
 6106 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 6107 %{
 6108   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6109   constraint(ALLOC_IN_RC(ptr_reg));
 6110   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 6111   op_cost(0);
 6112   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 6113   interface(MEMORY_INTER) %{
 6114     base($reg);
 6115     index($ireg);
 6116     scale($scale);
 6117     disp(0x0);
 6118   %}
 6119 %}
 6120 
 6121 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 6122 %{
 6123   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6124   constraint(ALLOC_IN_RC(ptr_reg));
 6125   match(AddP (DecodeN reg) (LShiftL lreg scale));
 6126   op_cost(0);
 6127   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 6128   interface(MEMORY_INTER) %{
 6129     base($reg);
 6130     index($lreg);
 6131     scale($scale);
 6132     disp(0x0);
 6133   %}
 6134 %}
 6135 
 6136 operand indIndexI2LN(iRegN reg, iRegI ireg)
 6137 %{
 6138   predicate(CompressedOops::shift() == 0);
 6139   constraint(ALLOC_IN_RC(ptr_reg));
 6140   match(AddP (DecodeN reg) (ConvI2L ireg));
 6141   op_cost(0);
 6142   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 6143   interface(MEMORY_INTER) %{
 6144     base($reg);
 6145     index($ireg);
 6146     scale(0x0);
 6147     disp(0x0);
 6148   %}
 6149 %}
 6150 
 6151 operand indIndexN(iRegN reg, iRegL lreg)
 6152 %{
 6153   predicate(CompressedOops::shift() == 0);
 6154   constraint(ALLOC_IN_RC(ptr_reg));
 6155   match(AddP (DecodeN reg) lreg);
 6156   op_cost(0);
 6157   format %{ "$reg, $lreg\t# narrow" %}
 6158   interface(MEMORY_INTER) %{
 6159     base($reg);
 6160     index($lreg);
 6161     scale(0x0);
 6162     disp(0x0);
 6163   %}
 6164 %}
 6165 
 6166 operand indOffIN(iRegN reg, immIOffset off)
 6167 %{
 6168   predicate(CompressedOops::shift() == 0);
 6169   constraint(ALLOC_IN_RC(ptr_reg));
 6170   match(AddP (DecodeN reg) off);
 6171   op_cost(0);
 6172   format %{ "[$reg, $off]\t# narrow" %}
 6173   interface(MEMORY_INTER) %{
 6174     base($reg);
 6175     index(0xffffffff);
 6176     scale(0x0);
 6177     disp($off);
 6178   %}
 6179 %}
 6180 
 6181 operand indOffLN(iRegN reg, immLoffset off)
 6182 %{
 6183   predicate(CompressedOops::shift() == 0);
 6184   constraint(ALLOC_IN_RC(ptr_reg));
 6185   match(AddP (DecodeN reg) off);
 6186   op_cost(0);
 6187   format %{ "[$reg, $off]\t# narrow" %}
 6188   interface(MEMORY_INTER) %{
 6189     base($reg);
 6190     index(0xffffffff);
 6191     scale(0x0);
 6192     disp($off);
 6193   %}
 6194 %}
 6195 
 6196 
 6197 
 6198 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6199 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6200 %{
 6201   constraint(ALLOC_IN_RC(ptr_reg));
 6202   match(AddP reg off);
 6203   op_cost(0);
 6204   format %{ "[$reg, $off]" %}
 6205   interface(MEMORY_INTER) %{
 6206     base($reg);
 6207     index(0xffffffff);
 6208     scale(0x0);
 6209     disp($off);
 6210   %}
 6211 %}
 6212 
 6213 //----------Special Memory Operands--------------------------------------------
 6214 // Stack Slot Operand - This operand is used for loading and storing temporary
 6215 //                      values on the stack where a match requires a value to
 6216 //                      flow through memory.
 6217 operand stackSlotP(sRegP reg)
 6218 %{
 6219   constraint(ALLOC_IN_RC(stack_slots));
 6220   op_cost(100);
 6221   // No match rule because this operand is only generated in matching
 6222   // match(RegP);
 6223   format %{ "[$reg]" %}
 6224   interface(MEMORY_INTER) %{
 6225     base(0x1e);  // RSP
 6226     index(0x0);  // No Index
 6227     scale(0x0);  // No Scale
 6228     disp($reg);  // Stack Offset
 6229   %}
 6230 %}
 6231 
 6232 operand stackSlotI(sRegI reg)
 6233 %{
 6234   constraint(ALLOC_IN_RC(stack_slots));
 6235   // No match rule because this operand is only generated in matching
 6236   // match(RegI);
 6237   format %{ "[$reg]" %}
 6238   interface(MEMORY_INTER) %{
 6239     base(0x1e);  // RSP
 6240     index(0x0);  // No Index
 6241     scale(0x0);  // No Scale
 6242     disp($reg);  // Stack Offset
 6243   %}
 6244 %}
 6245 
 6246 operand stackSlotF(sRegF reg)
 6247 %{
 6248   constraint(ALLOC_IN_RC(stack_slots));
 6249   // No match rule because this operand is only generated in matching
 6250   // match(RegF);
 6251   format %{ "[$reg]" %}
 6252   interface(MEMORY_INTER) %{
 6253     base(0x1e);  // RSP
 6254     index(0x0);  // No Index
 6255     scale(0x0);  // No Scale
 6256     disp($reg);  // Stack Offset
 6257   %}
 6258 %}
 6259 
 6260 operand stackSlotD(sRegD reg)
 6261 %{
 6262   constraint(ALLOC_IN_RC(stack_slots));
 6263   // No match rule because this operand is only generated in matching
 6264   // match(RegD);
 6265   format %{ "[$reg]" %}
 6266   interface(MEMORY_INTER) %{
 6267     base(0x1e);  // RSP
 6268     index(0x0);  // No Index
 6269     scale(0x0);  // No Scale
 6270     disp($reg);  // Stack Offset
 6271   %}
 6272 %}
 6273 
 6274 operand stackSlotL(sRegL reg)
 6275 %{
 6276   constraint(ALLOC_IN_RC(stack_slots));
 6277   // No match rule because this operand is only generated in matching
 6278   // match(RegL);
 6279   format %{ "[$reg]" %}
 6280   interface(MEMORY_INTER) %{
 6281     base(0x1e);  // RSP
 6282     index(0x0);  // No Index
 6283     scale(0x0);  // No Scale
 6284     disp($reg);  // Stack Offset
 6285   %}
 6286 %}
 6287 
 6288 // Operands for expressing Control Flow
 6289 // NOTE: Label is a predefined operand which should not be redefined in
 6290 //       the AD file. It is generically handled within the ADLC.
 6291 
 6292 //----------Conditional Branch Operands----------------------------------------
 6293 // Comparison Op  - This is the operation of the comparison, and is limited to
 6294 //                  the following set of codes:
 6295 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6296 //
 6297 // Other attributes of the comparison, such as unsignedness, are specified
 6298 // by the comparison instruction that sets a condition code flags register.
 6299 // That result is represented by a flags operand whose subtype is appropriate
 6300 // to the unsignedness (etc.) of the comparison.
 6301 //
 6302 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6303 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6304 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6305 
 6306 // used for signed integral comparisons and fp comparisons
 6307 
 6308 operand cmpOp()
 6309 %{
 6310   match(Bool);
 6311 
 6312   format %{ "" %}
 6313   interface(COND_INTER) %{
 6314     equal(0x0, "eq");
 6315     not_equal(0x1, "ne");
 6316     less(0xb, "lt");
 6317     greater_equal(0xa, "ge");
 6318     less_equal(0xd, "le");
 6319     greater(0xc, "gt");
 6320     overflow(0x6, "vs");
 6321     no_overflow(0x7, "vc");
 6322   %}
 6323 %}
 6324 
 6325 // used for unsigned integral comparisons
 6326 
 6327 operand cmpOpU()
 6328 %{
 6329   match(Bool);
 6330 
 6331   format %{ "" %}
 6332   interface(COND_INTER) %{
 6333     equal(0x0, "eq");
 6334     not_equal(0x1, "ne");
 6335     less(0x3, "lo");
 6336     greater_equal(0x2, "hs");
 6337     less_equal(0x9, "ls");
 6338     greater(0x8, "hi");
 6339     overflow(0x6, "vs");
 6340     no_overflow(0x7, "vc");
 6341   %}
 6342 %}
 6343 
 6344 // used for certain integral comparisons which can be
 6345 // converted to cbxx or tbxx instructions
 6346 
 6347 operand cmpOpEqNe()
 6348 %{
 6349   match(Bool);
 6350   op_cost(0);
 6351   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6352             || n->as_Bool()->_test._test == BoolTest::eq);
 6353 
 6354   format %{ "" %}
 6355   interface(COND_INTER) %{
 6356     equal(0x0, "eq");
 6357     not_equal(0x1, "ne");
 6358     less(0xb, "lt");
 6359     greater_equal(0xa, "ge");
 6360     less_equal(0xd, "le");
 6361     greater(0xc, "gt");
 6362     overflow(0x6, "vs");
 6363     no_overflow(0x7, "vc");
 6364   %}
 6365 %}
 6366 
 6367 // used for certain integral comparisons which can be
 6368 // converted to cbxx or tbxx instructions
 6369 
 6370 operand cmpOpLtGe()
 6371 %{
 6372   match(Bool);
 6373   op_cost(0);
 6374 
 6375   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6376             || n->as_Bool()->_test._test == BoolTest::ge);
 6377 
 6378   format %{ "" %}
 6379   interface(COND_INTER) %{
 6380     equal(0x0, "eq");
 6381     not_equal(0x1, "ne");
 6382     less(0xb, "lt");
 6383     greater_equal(0xa, "ge");
 6384     less_equal(0xd, "le");
 6385     greater(0xc, "gt");
 6386     overflow(0x6, "vs");
 6387     no_overflow(0x7, "vc");
 6388   %}
 6389 %}
 6390 
 6391 // used for certain unsigned integral comparisons which can be
 6392 // converted to cbxx or tbxx instructions
 6393 
 6394 operand cmpOpUEqNeLtGe()
 6395 %{
 6396   match(Bool);
 6397   op_cost(0);
 6398 
 6399   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6400             || n->as_Bool()->_test._test == BoolTest::ne
 6401             || n->as_Bool()->_test._test == BoolTest::lt
 6402             || n->as_Bool()->_test._test == BoolTest::ge);
 6403 
 6404   format %{ "" %}
 6405   interface(COND_INTER) %{
 6406     equal(0x0, "eq");
 6407     not_equal(0x1, "ne");
 6408     less(0xb, "lt");
 6409     greater_equal(0xa, "ge");
 6410     less_equal(0xd, "le");
 6411     greater(0xc, "gt");
 6412     overflow(0x6, "vs");
 6413     no_overflow(0x7, "vc");
 6414   %}
 6415 %}
 6416 
 6417 // Special operand allowing long args to int ops to be truncated for free
 6418 
 6419 operand iRegL2I(iRegL reg) %{
 6420 
 6421   op_cost(0);
 6422 
 6423   match(ConvL2I reg);
 6424 
 6425   format %{ "l2i($reg)" %}
 6426 
 6427   interface(REG_INTER)
 6428 %}
 6429 
 6430 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6431 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6432 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6433 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6434 
 6435 //----------OPERAND CLASSES----------------------------------------------------
 6436 // Operand Classes are groups of operands that are used as to simplify
 6437 // instruction definitions by not requiring the AD writer to specify
 6438 // separate instructions for every form of operand when the
 6439 // instruction accepts multiple operand types with the same basic
 6440 // encoding and format. The classic case of this is memory operands.
 6441 
 6442 // memory is used to define read/write location for load/store
 6443 // instruction defs. we can turn a memory op into an Address
 6444 
 6445 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6446                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6447 
 6448 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6449                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6450 
 6451 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6452                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6453 
 6454 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6455                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6456 
 6457 // All of the memory operands. For the pipeline description.
 6458 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6459                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6460                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6461 
 6462 
 6463 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6464 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6465 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6466 // can be elided because the 32-bit instruction will just employ the
 6467 // lower 32 bits anyway.
 6468 //
 6469 // n.b. this does not elide all L2I conversions. if the truncated
 6470 // value is consumed by more than one operation then the ConvL2I
 6471 // cannot be bundled into the consuming nodes so an l2i gets planted
 6472 // (actually a movw $dst $src) and the downstream instructions consume
 6473 // the result of the l2i as an iRegI input. That's a shame since the
 6474 // movw is actually redundant but its not too costly.
 6475 
 6476 opclass iRegIorL2I(iRegI, iRegL2I);
 6477 
 6478 //----------PIPELINE-----------------------------------------------------------
 6479 // Rules which define the behavior of the target architectures pipeline.
 6480 
 6481 // For specific pipelines, eg A53, define the stages of that pipeline
 6482 //pipe_desc(ISS, EX1, EX2, WR);
 6483 #define ISS S0
 6484 #define EX1 S1
 6485 #define EX2 S2
 6486 #define WR  S3
 6487 
 6488 // Integer ALU reg operation
 6489 pipeline %{
 6490 
 6491 attributes %{
 6492   // ARM instructions are of fixed length
 6493   fixed_size_instructions;        // Fixed size instructions TODO does
 6494   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6495   // ARM instructions come in 32-bit word units
 6496   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6497   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6498   instruction_fetch_units = 1;       // of 64 bytes
 6499 
 6500   // List of nop instructions
 6501   nops( MachNop );
 6502 %}
 6503 
 6504 // We don't use an actual pipeline model so don't care about resources
 6505 // or description. we do use pipeline classes to introduce fixed
 6506 // latencies
 6507 
 6508 //----------RESOURCES----------------------------------------------------------
 6509 // Resources are the functional units available to the machine
 6510 
 6511 resources( INS0, INS1, INS01 = INS0 | INS1,
 6512            ALU0, ALU1, ALU = ALU0 | ALU1,
 6513            MAC,
 6514            DIV,
 6515            BRANCH,
 6516            LDST,
 6517            NEON_FP);
 6518 
 6519 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6520 // Pipeline Description specifies the stages in the machine's pipeline
 6521 
 6522 // Define the pipeline as a generic 6 stage pipeline
 6523 pipe_desc(S0, S1, S2, S3, S4, S5);
 6524 
 6525 //----------PIPELINE CLASSES---------------------------------------------------
 6526 // Pipeline Classes describe the stages in which input and output are
 6527 // referenced by the hardware pipeline.
 6528 
 6529 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6530 %{
 6531   single_instruction;
 6532   src1   : S1(read);
 6533   src2   : S2(read);
 6534   dst    : S5(write);
 6535   INS01  : ISS;
 6536   NEON_FP : S5;
 6537 %}
 6538 
 6539 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6540 %{
 6541   single_instruction;
 6542   src1   : S1(read);
 6543   src2   : S2(read);
 6544   dst    : S5(write);
 6545   INS01  : ISS;
 6546   NEON_FP : S5;
 6547 %}
 6548 
 6549 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6550 %{
 6551   single_instruction;
 6552   src    : S1(read);
 6553   dst    : S5(write);
 6554   INS01  : ISS;
 6555   NEON_FP : S5;
 6556 %}
 6557 
 6558 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6559 %{
 6560   single_instruction;
 6561   src    : S1(read);
 6562   dst    : S5(write);
 6563   INS01  : ISS;
 6564   NEON_FP : S5;
 6565 %}
 6566 
 6567 pipe_class fp_d2f(vRegF dst, vRegD src)
 6568 %{
 6569   single_instruction;
 6570   src    : S1(read);
 6571   dst    : S5(write);
 6572   INS01  : ISS;
 6573   NEON_FP : S5;
 6574 %}
 6575 
 6576 pipe_class fp_f2d(vRegD dst, vRegF src)
 6577 %{
 6578   single_instruction;
 6579   src    : S1(read);
 6580   dst    : S5(write);
 6581   INS01  : ISS;
 6582   NEON_FP : S5;
 6583 %}
 6584 
 6585 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6586 %{
 6587   single_instruction;
 6588   src    : S1(read);
 6589   dst    : S5(write);
 6590   INS01  : ISS;
 6591   NEON_FP : S5;
 6592 %}
 6593 
 6594 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6595 %{
 6596   single_instruction;
 6597   src    : S1(read);
 6598   dst    : S5(write);
 6599   INS01  : ISS;
 6600   NEON_FP : S5;
 6601 %}
 6602 
 6603 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6604 %{
 6605   single_instruction;
 6606   src    : S1(read);
 6607   dst    : S5(write);
 6608   INS01  : ISS;
 6609   NEON_FP : S5;
 6610 %}
 6611 
 6612 pipe_class fp_l2f(vRegF dst, iRegL src)
 6613 %{
 6614   single_instruction;
 6615   src    : S1(read);
 6616   dst    : S5(write);
 6617   INS01  : ISS;
 6618   NEON_FP : S5;
 6619 %}
 6620 
 6621 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6622 %{
 6623   single_instruction;
 6624   src    : S1(read);
 6625   dst    : S5(write);
 6626   INS01  : ISS;
 6627   NEON_FP : S5;
 6628 %}
 6629 
 6630 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6631 %{
 6632   single_instruction;
 6633   src    : S1(read);
 6634   dst    : S5(write);
 6635   INS01  : ISS;
 6636   NEON_FP : S5;
 6637 %}
 6638 
 6639 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6640 %{
 6641   single_instruction;
 6642   src    : S1(read);
 6643   dst    : S5(write);
 6644   INS01  : ISS;
 6645   NEON_FP : S5;
 6646 %}
 6647 
 6648 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6649 %{
 6650   single_instruction;
 6651   src    : S1(read);
 6652   dst    : S5(write);
 6653   INS01  : ISS;
 6654   NEON_FP : S5;
 6655 %}
 6656 
 6657 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6658 %{
 6659   single_instruction;
 6660   src1   : S1(read);
 6661   src2   : S2(read);
 6662   dst    : S5(write);
 6663   INS0   : ISS;
 6664   NEON_FP : S5;
 6665 %}
 6666 
 6667 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6668 %{
 6669   single_instruction;
 6670   src1   : S1(read);
 6671   src2   : S2(read);
 6672   dst    : S5(write);
 6673   INS0   : ISS;
 6674   NEON_FP : S5;
 6675 %}
 6676 
 6677 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6678 %{
 6679   single_instruction;
 6680   cr     : S1(read);
 6681   src1   : S1(read);
 6682   src2   : S1(read);
 6683   dst    : S3(write);
 6684   INS01  : ISS;
 6685   NEON_FP : S3;
 6686 %}
 6687 
 6688 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6689 %{
 6690   single_instruction;
 6691   cr     : S1(read);
 6692   src1   : S1(read);
 6693   src2   : S1(read);
 6694   dst    : S3(write);
 6695   INS01  : ISS;
 6696   NEON_FP : S3;
 6697 %}
 6698 
 6699 pipe_class fp_imm_s(vRegF dst)
 6700 %{
 6701   single_instruction;
 6702   dst    : S3(write);
 6703   INS01  : ISS;
 6704   NEON_FP : S3;
 6705 %}
 6706 
 6707 pipe_class fp_imm_d(vRegD dst)
 6708 %{
 6709   single_instruction;
 6710   dst    : S3(write);
 6711   INS01  : ISS;
 6712   NEON_FP : S3;
 6713 %}
 6714 
 6715 pipe_class fp_load_constant_s(vRegF dst)
 6716 %{
 6717   single_instruction;
 6718   dst    : S4(write);
 6719   INS01  : ISS;
 6720   NEON_FP : S4;
 6721 %}
 6722 
 6723 pipe_class fp_load_constant_d(vRegD dst)
 6724 %{
 6725   single_instruction;
 6726   dst    : S4(write);
 6727   INS01  : ISS;
 6728   NEON_FP : S4;
 6729 %}
 6730 
 6731 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6732 %{
 6733   single_instruction;
 6734   dst    : S5(write);
 6735   src1   : S1(read);
 6736   src2   : S1(read);
 6737   INS01  : ISS;
 6738   NEON_FP : S5;
 6739 %}
 6740 
 6741 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6742 %{
 6743   single_instruction;
 6744   dst    : S5(write);
 6745   src1   : S1(read);
 6746   src2   : S1(read);
 6747   INS0   : ISS;
 6748   NEON_FP : S5;
 6749 %}
 6750 
 6751 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6752 %{
 6753   single_instruction;
 6754   dst    : S5(write);
 6755   src1   : S1(read);
 6756   src2   : S1(read);
 6757   dst    : S1(read);
 6758   INS01  : ISS;
 6759   NEON_FP : S5;
 6760 %}
 6761 
 6762 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6763 %{
 6764   single_instruction;
 6765   dst    : S5(write);
 6766   src1   : S1(read);
 6767   src2   : S1(read);
 6768   dst    : S1(read);
 6769   INS0   : ISS;
 6770   NEON_FP : S5;
 6771 %}
 6772 
 6773 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6774 %{
 6775   single_instruction;
 6776   dst    : S4(write);
 6777   src1   : S2(read);
 6778   src2   : S2(read);
 6779   INS01  : ISS;
 6780   NEON_FP : S4;
 6781 %}
 6782 
 6783 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6784 %{
 6785   single_instruction;
 6786   dst    : S4(write);
 6787   src1   : S2(read);
 6788   src2   : S2(read);
 6789   INS0   : ISS;
 6790   NEON_FP : S4;
 6791 %}
 6792 
 6793 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6794 %{
 6795   single_instruction;
 6796   dst    : S3(write);
 6797   src1   : S2(read);
 6798   src2   : S2(read);
 6799   INS01  : ISS;
 6800   NEON_FP : S3;
 6801 %}
 6802 
 6803 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6804 %{
 6805   single_instruction;
 6806   dst    : S3(write);
 6807   src1   : S2(read);
 6808   src2   : S2(read);
 6809   INS0   : ISS;
 6810   NEON_FP : S3;
 6811 %}
 6812 
 6813 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6814 %{
 6815   single_instruction;
 6816   dst    : S3(write);
 6817   src    : S1(read);
 6818   shift  : S1(read);
 6819   INS01  : ISS;
 6820   NEON_FP : S3;
 6821 %}
 6822 
 6823 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6824 %{
 6825   single_instruction;
 6826   dst    : S3(write);
 6827   src    : S1(read);
 6828   shift  : S1(read);
 6829   INS0   : ISS;
 6830   NEON_FP : S3;
 6831 %}
 6832 
 6833 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6834 %{
 6835   single_instruction;
 6836   dst    : S3(write);
 6837   src    : S1(read);
 6838   INS01  : ISS;
 6839   NEON_FP : S3;
 6840 %}
 6841 
 6842 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6843 %{
 6844   single_instruction;
 6845   dst    : S3(write);
 6846   src    : S1(read);
 6847   INS0   : ISS;
 6848   NEON_FP : S3;
 6849 %}
 6850 
 6851 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6852 %{
 6853   single_instruction;
 6854   dst    : S5(write);
 6855   src1   : S1(read);
 6856   src2   : S1(read);
 6857   INS01  : ISS;
 6858   NEON_FP : S5;
 6859 %}
 6860 
 6861 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6862 %{
 6863   single_instruction;
 6864   dst    : S5(write);
 6865   src1   : S1(read);
 6866   src2   : S1(read);
 6867   INS0   : ISS;
 6868   NEON_FP : S5;
 6869 %}
 6870 
 6871 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6872 %{
 6873   single_instruction;
 6874   dst    : S5(write);
 6875   src1   : S1(read);
 6876   src2   : S1(read);
 6877   INS0   : ISS;
 6878   NEON_FP : S5;
 6879 %}
 6880 
 6881 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6882 %{
 6883   single_instruction;
 6884   dst    : S5(write);
 6885   src1   : S1(read);
 6886   src2   : S1(read);
 6887   INS0   : ISS;
 6888   NEON_FP : S5;
 6889 %}
 6890 
 6891 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6892 %{
 6893   single_instruction;
 6894   dst    : S5(write);
 6895   src    : S1(read);
 6896   INS0   : ISS;
 6897   NEON_FP : S5;
 6898 %}
 6899 
 6900 pipe_class vunop_fp64(vecD dst, vecD src)
 6901 %{
 6902   single_instruction;
 6903   dst    : S5(write);
 6904   src    : S1(read);
 6905   INS01  : ISS;
 6906   NEON_FP : S5;
 6907 %}
 6908 
 6909 pipe_class vunop_fp128(vecX dst, vecX src)
 6910 %{
 6911   single_instruction;
 6912   dst    : S5(write);
 6913   src    : S1(read);
 6914   INS0   : ISS;
 6915   NEON_FP : S5;
 6916 %}
 6917 
 6918 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6919 %{
 6920   single_instruction;
 6921   dst    : S3(write);
 6922   src    : S1(read);
 6923   INS01  : ISS;
 6924   NEON_FP : S3;
 6925 %}
 6926 
 6927 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6928 %{
 6929   single_instruction;
 6930   dst    : S3(write);
 6931   src    : S1(read);
 6932   INS01  : ISS;
 6933   NEON_FP : S3;
 6934 %}
 6935 
 6936 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6937 %{
 6938   single_instruction;
 6939   dst    : S3(write);
 6940   src    : S1(read);
 6941   INS01  : ISS;
 6942   NEON_FP : S3;
 6943 %}
 6944 
 6945 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6946 %{
 6947   single_instruction;
 6948   dst    : S3(write);
 6949   src    : S1(read);
 6950   INS01  : ISS;
 6951   NEON_FP : S3;
 6952 %}
 6953 
 6954 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6955 %{
 6956   single_instruction;
 6957   dst    : S3(write);
 6958   src    : S1(read);
 6959   INS01  : ISS;
 6960   NEON_FP : S3;
 6961 %}
 6962 
 6963 pipe_class vmovi_reg_imm64(vecD dst)
 6964 %{
 6965   single_instruction;
 6966   dst    : S3(write);
 6967   INS01  : ISS;
 6968   NEON_FP : S3;
 6969 %}
 6970 
 6971 pipe_class vmovi_reg_imm128(vecX dst)
 6972 %{
 6973   single_instruction;
 6974   dst    : S3(write);
 6975   INS0   : ISS;
 6976   NEON_FP : S3;
 6977 %}
 6978 
 6979 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6980 %{
 6981   single_instruction;
 6982   dst    : S5(write);
 6983   mem    : ISS(read);
 6984   INS01  : ISS;
 6985   NEON_FP : S3;
 6986 %}
 6987 
 6988 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6989 %{
 6990   single_instruction;
 6991   dst    : S5(write);
 6992   mem    : ISS(read);
 6993   INS01  : ISS;
 6994   NEON_FP : S3;
 6995 %}
 6996 
 6997 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6998 %{
 6999   single_instruction;
 7000   mem    : ISS(read);
 7001   src    : S2(read);
 7002   INS01  : ISS;
 7003   NEON_FP : S3;
 7004 %}
 7005 
 7006 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 7007 %{
 7008   single_instruction;
 7009   mem    : ISS(read);
 7010   src    : S2(read);
 7011   INS01  : ISS;
 7012   NEON_FP : S3;
 7013 %}
 7014 
 7015 //------- Integer ALU operations --------------------------
 7016 
 7017 // Integer ALU reg-reg operation
 7018 // Operands needed in EX1, result generated in EX2
 7019 // Eg.  ADD     x0, x1, x2
 7020 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7021 %{
 7022   single_instruction;
 7023   dst    : EX2(write);
 7024   src1   : EX1(read);
 7025   src2   : EX1(read);
 7026   INS01  : ISS; // Dual issue as instruction 0 or 1
 7027   ALU    : EX2;
 7028 %}
 7029 
 7030 // Integer ALU reg-reg operation with constant shift
 7031 // Shifted register must be available in LATE_ISS instead of EX1
 7032 // Eg.  ADD     x0, x1, x2, LSL #2
 7033 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 7034 %{
 7035   single_instruction;
 7036   dst    : EX2(write);
 7037   src1   : EX1(read);
 7038   src2   : ISS(read);
 7039   INS01  : ISS;
 7040   ALU    : EX2;
 7041 %}
 7042 
 7043 // Integer ALU reg operation with constant shift
 7044 // Eg.  LSL     x0, x1, #shift
 7045 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 7046 %{
 7047   single_instruction;
 7048   dst    : EX2(write);
 7049   src1   : ISS(read);
 7050   INS01  : ISS;
 7051   ALU    : EX2;
 7052 %}
 7053 
 7054 // Integer ALU reg-reg operation with variable shift
 7055 // Both operands must be available in LATE_ISS instead of EX1
 7056 // Result is available in EX1 instead of EX2
 7057 // Eg.  LSLV    x0, x1, x2
 7058 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 7059 %{
 7060   single_instruction;
 7061   dst    : EX1(write);
 7062   src1   : ISS(read);
 7063   src2   : ISS(read);
 7064   INS01  : ISS;
 7065   ALU    : EX1;
 7066 %}
 7067 
 7068 // Integer ALU reg-reg operation with extract
 7069 // As for _vshift above, but result generated in EX2
 7070 // Eg.  EXTR    x0, x1, x2, #N
 7071 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 7072 %{
 7073   single_instruction;
 7074   dst    : EX2(write);
 7075   src1   : ISS(read);
 7076   src2   : ISS(read);
 7077   INS1   : ISS; // Can only dual issue as Instruction 1
 7078   ALU    : EX1;
 7079 %}
 7080 
 7081 // Integer ALU reg operation
 7082 // Eg.  NEG     x0, x1
 7083 pipe_class ialu_reg(iRegI dst, iRegI src)
 7084 %{
 7085   single_instruction;
 7086   dst    : EX2(write);
 7087   src    : EX1(read);
 7088   INS01  : ISS;
 7089   ALU    : EX2;
 7090 %}
 7091 
 7092 // Integer ALU reg mmediate operation
 7093 // Eg.  ADD     x0, x1, #N
 7094 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 7095 %{
 7096   single_instruction;
 7097   dst    : EX2(write);
 7098   src1   : EX1(read);
 7099   INS01  : ISS;
 7100   ALU    : EX2;
 7101 %}
 7102 
 7103 // Integer ALU immediate operation (no source operands)
 7104 // Eg.  MOV     x0, #N
 7105 pipe_class ialu_imm(iRegI dst)
 7106 %{
 7107   single_instruction;
 7108   dst    : EX1(write);
 7109   INS01  : ISS;
 7110   ALU    : EX1;
 7111 %}
 7112 
 7113 //------- Compare operation -------------------------------
 7114 
 7115 // Compare reg-reg
 7116 // Eg.  CMP     x0, x1
 7117 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 7118 %{
 7119   single_instruction;
 7120 //  fixed_latency(16);
 7121   cr     : EX2(write);
 7122   op1    : EX1(read);
 7123   op2    : EX1(read);
 7124   INS01  : ISS;
 7125   ALU    : EX2;
 7126 %}
 7127 
 7128 // Compare reg-reg
 7129 // Eg.  CMP     x0, #N
 7130 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 7131 %{
 7132   single_instruction;
 7133 //  fixed_latency(16);
 7134   cr     : EX2(write);
 7135   op1    : EX1(read);
 7136   INS01  : ISS;
 7137   ALU    : EX2;
 7138 %}
 7139 
 7140 //------- Conditional instructions ------------------------
 7141 
 7142 // Conditional no operands
 7143 // Eg.  CSINC   x0, zr, zr, <cond>
 7144 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 7145 %{
 7146   single_instruction;
 7147   cr     : EX1(read);
 7148   dst    : EX2(write);
 7149   INS01  : ISS;
 7150   ALU    : EX2;
 7151 %}
 7152 
 7153 // Conditional 2 operand
 7154 // EG.  CSEL    X0, X1, X2, <cond>
 7155 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 7156 %{
 7157   single_instruction;
 7158   cr     : EX1(read);
 7159   src1   : EX1(read);
 7160   src2   : EX1(read);
 7161   dst    : EX2(write);
 7162   INS01  : ISS;
 7163   ALU    : EX2;
 7164 %}
 7165 
 7166 // Conditional 2 operand
 7167 // EG.  CSEL    X0, X1, X2, <cond>
 7168 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 7169 %{
 7170   single_instruction;
 7171   cr     : EX1(read);
 7172   src    : EX1(read);
 7173   dst    : EX2(write);
 7174   INS01  : ISS;
 7175   ALU    : EX2;
 7176 %}
 7177 
 7178 //------- Multiply pipeline operations --------------------
 7179 
 7180 // Multiply reg-reg
 7181 // Eg.  MUL     w0, w1, w2
 7182 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7183 %{
 7184   single_instruction;
 7185   dst    : WR(write);
 7186   src1   : ISS(read);
 7187   src2   : ISS(read);
 7188   INS01  : ISS;
 7189   MAC    : WR;
 7190 %}
 7191 
 7192 // Multiply accumulate
 7193 // Eg.  MADD    w0, w1, w2, w3
 7194 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7195 %{
 7196   single_instruction;
 7197   dst    : WR(write);
 7198   src1   : ISS(read);
 7199   src2   : ISS(read);
 7200   src3   : ISS(read);
 7201   INS01  : ISS;
 7202   MAC    : WR;
 7203 %}
 7204 
 7205 // Eg.  MUL     w0, w1, w2
 7206 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7207 %{
 7208   single_instruction;
 7209   fixed_latency(3); // Maximum latency for 64 bit mul
 7210   dst    : WR(write);
 7211   src1   : ISS(read);
 7212   src2   : ISS(read);
 7213   INS01  : ISS;
 7214   MAC    : WR;
 7215 %}
 7216 
 7217 // Multiply accumulate
 7218 // Eg.  MADD    w0, w1, w2, w3
 7219 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7220 %{
 7221   single_instruction;
 7222   fixed_latency(3); // Maximum latency for 64 bit mul
 7223   dst    : WR(write);
 7224   src1   : ISS(read);
 7225   src2   : ISS(read);
 7226   src3   : ISS(read);
 7227   INS01  : ISS;
 7228   MAC    : WR;
 7229 %}
 7230 
 7231 //------- Divide pipeline operations --------------------
 7232 
 7233 // Eg.  SDIV    w0, w1, w2
 7234 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7235 %{
 7236   single_instruction;
 7237   fixed_latency(8); // Maximum latency for 32 bit divide
 7238   dst    : WR(write);
 7239   src1   : ISS(read);
 7240   src2   : ISS(read);
 7241   INS0   : ISS; // Can only dual issue as instruction 0
 7242   DIV    : WR;
 7243 %}
 7244 
 7245 // Eg.  SDIV    x0, x1, x2
 7246 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7247 %{
 7248   single_instruction;
 7249   fixed_latency(16); // Maximum latency for 64 bit divide
 7250   dst    : WR(write);
 7251   src1   : ISS(read);
 7252   src2   : ISS(read);
 7253   INS0   : ISS; // Can only dual issue as instruction 0
 7254   DIV    : WR;
 7255 %}
 7256 
 7257 //------- Load pipeline operations ------------------------
 7258 
 7259 // Load - prefetch
 7260 // Eg.  PFRM    <mem>
 7261 pipe_class iload_prefetch(memory mem)
 7262 %{
 7263   single_instruction;
 7264   mem    : ISS(read);
 7265   INS01  : ISS;
 7266   LDST   : WR;
 7267 %}
 7268 
 7269 // Load - reg, mem
 7270 // Eg.  LDR     x0, <mem>
 7271 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7272 %{
 7273   single_instruction;
 7274   dst    : WR(write);
 7275   mem    : ISS(read);
 7276   INS01  : ISS;
 7277   LDST   : WR;
 7278 %}
 7279 
 7280 // Load - reg, reg
 7281 // Eg.  LDR     x0, [sp, x1]
 7282 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7283 %{
 7284   single_instruction;
 7285   dst    : WR(write);
 7286   src    : ISS(read);
 7287   INS01  : ISS;
 7288   LDST   : WR;
 7289 %}
 7290 
 7291 //------- Store pipeline operations -----------------------
 7292 
 7293 // Store - zr, mem
 7294 // Eg.  STR     zr, <mem>
 7295 pipe_class istore_mem(memory mem)
 7296 %{
 7297   single_instruction;
 7298   mem    : ISS(read);
 7299   INS01  : ISS;
 7300   LDST   : WR;
 7301 %}
 7302 
 7303 // Store - reg, mem
 7304 // Eg.  STR     x0, <mem>
 7305 pipe_class istore_reg_mem(iRegI src, memory mem)
 7306 %{
 7307   single_instruction;
 7308   mem    : ISS(read);
 7309   src    : EX2(read);
 7310   INS01  : ISS;
 7311   LDST   : WR;
 7312 %}
 7313 
 7314 // Store - reg, reg
 7315 // Eg. STR      x0, [sp, x1]
 7316 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7317 %{
 7318   single_instruction;
 7319   dst    : ISS(read);
 7320   src    : EX2(read);
 7321   INS01  : ISS;
 7322   LDST   : WR;
 7323 %}
 7324 
 7325 //------- Store pipeline operations -----------------------
 7326 
 7327 // Branch
 7328 pipe_class pipe_branch()
 7329 %{
 7330   single_instruction;
 7331   INS01  : ISS;
 7332   BRANCH : EX1;
 7333 %}
 7334 
 7335 // Conditional branch
 7336 pipe_class pipe_branch_cond(rFlagsReg cr)
 7337 %{
 7338   single_instruction;
 7339   cr     : EX1(read);
 7340   INS01  : ISS;
 7341   BRANCH : EX1;
 7342 %}
 7343 
 7344 // Compare & Branch
 7345 // EG.  CBZ/CBNZ
 7346 pipe_class pipe_cmp_branch(iRegI op1)
 7347 %{
 7348   single_instruction;
 7349   op1    : EX1(read);
 7350   INS01  : ISS;
 7351   BRANCH : EX1;
 7352 %}
 7353 
 7354 //------- Synchronisation operations ----------------------
 7355 
 7356 // Any operation requiring serialization.
 7357 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7358 pipe_class pipe_serial()
 7359 %{
 7360   single_instruction;
 7361   force_serialization;
 7362   fixed_latency(16);
 7363   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7364   LDST   : WR;
 7365 %}
 7366 
 7367 // Generic big/slow expanded idiom - also serialized
 7368 pipe_class pipe_slow()
 7369 %{
 7370   instruction_count(10);
 7371   multiple_bundles;
 7372   force_serialization;
 7373   fixed_latency(16);
 7374   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7375   LDST   : WR;
 7376 %}
 7377 
 7378 // Empty pipeline class
 7379 pipe_class pipe_class_empty()
 7380 %{
 7381   single_instruction;
 7382   fixed_latency(0);
 7383 %}
 7384 
 7385 // Default pipeline class.
 7386 pipe_class pipe_class_default()
 7387 %{
 7388   single_instruction;
 7389   fixed_latency(2);
 7390 %}
 7391 
 7392 // Pipeline class for compares.
 7393 pipe_class pipe_class_compare()
 7394 %{
 7395   single_instruction;
 7396   fixed_latency(16);
 7397 %}
 7398 
 7399 // Pipeline class for memory operations.
 7400 pipe_class pipe_class_memory()
 7401 %{
 7402   single_instruction;
 7403   fixed_latency(16);
 7404 %}
 7405 
 7406 // Pipeline class for call.
 7407 pipe_class pipe_class_call()
 7408 %{
 7409   single_instruction;
 7410   fixed_latency(100);
 7411 %}
 7412 
 7413 // Define the class for the Nop node.
 7414 define %{
 7415    MachNop = pipe_class_empty;
 7416 %}
 7417 
 7418 %}
 7419 //----------INSTRUCTIONS-------------------------------------------------------
 7420 //
 7421 // match      -- States which machine-independent subtree may be replaced
 7422 //               by this instruction.
 7423 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7424 //               selection to identify a minimum cost tree of machine
 7425 //               instructions that matches a tree of machine-independent
 7426 //               instructions.
 7427 // format     -- A string providing the disassembly for this instruction.
 7428 //               The value of an instruction's operand may be inserted
 7429 //               by referring to it with a '$' prefix.
 7430 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7431 //               to within an encode class as $primary, $secondary, and $tertiary
 7432 //               rrspectively.  The primary opcode is commonly used to
 7433 //               indicate the type of machine instruction, while secondary
 7434 //               and tertiary are often used for prefix options or addressing
 7435 //               modes.
 7436 // ins_encode -- A list of encode classes with parameters. The encode class
 7437 //               name must have been defined in an 'enc_class' specification
 7438 //               in the encode section of the architecture description.
 7439 
 7440 // ============================================================================
 7441 // Memory (Load/Store) Instructions
 7442 
 7443 // Load Instructions
 7444 
 7445 // Load Byte (8 bit signed)
 7446 instruct loadB(iRegINoSp dst, memory1 mem)
 7447 %{
 7448   match(Set dst (LoadB mem));
 7449   predicate(!needs_acquiring_load(n));
 7450 
 7451   ins_cost(4 * INSN_COST);
 7452   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7453 
 7454   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7455 
 7456   ins_pipe(iload_reg_mem);
 7457 %}
 7458 
 7459 // Load Byte (8 bit signed) into long
 7460 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7461 %{
 7462   match(Set dst (ConvI2L (LoadB mem)));
 7463   predicate(!needs_acquiring_load(n->in(1)));
 7464 
 7465   ins_cost(4 * INSN_COST);
 7466   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7467 
 7468   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7469 
 7470   ins_pipe(iload_reg_mem);
 7471 %}
 7472 
 7473 // Load Byte (8 bit unsigned)
 7474 instruct loadUB(iRegINoSp dst, memory1 mem)
 7475 %{
 7476   match(Set dst (LoadUB mem));
 7477   predicate(!needs_acquiring_load(n));
 7478 
 7479   ins_cost(4 * INSN_COST);
 7480   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7481 
 7482   ins_encode(aarch64_enc_ldrb(dst, mem));
 7483 
 7484   ins_pipe(iload_reg_mem);
 7485 %}
 7486 
 7487 // Load Byte (8 bit unsigned) into long
 7488 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7489 %{
 7490   match(Set dst (ConvI2L (LoadUB mem)));
 7491   predicate(!needs_acquiring_load(n->in(1)));
 7492 
 7493   ins_cost(4 * INSN_COST);
 7494   format %{ "ldrb  $dst, $mem\t# byte" %}
 7495 
 7496   ins_encode(aarch64_enc_ldrb(dst, mem));
 7497 
 7498   ins_pipe(iload_reg_mem);
 7499 %}
 7500 
 7501 // Load Short (16 bit signed)
 7502 instruct loadS(iRegINoSp dst, memory2 mem)
 7503 %{
 7504   match(Set dst (LoadS mem));
 7505   predicate(!needs_acquiring_load(n));
 7506 
 7507   ins_cost(4 * INSN_COST);
 7508   format %{ "ldrshw  $dst, $mem\t# short" %}
 7509 
 7510   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7511 
 7512   ins_pipe(iload_reg_mem);
 7513 %}
 7514 
 7515 // Load Short (16 bit signed) into long
 7516 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7517 %{
 7518   match(Set dst (ConvI2L (LoadS mem)));
 7519   predicate(!needs_acquiring_load(n->in(1)));
 7520 
 7521   ins_cost(4 * INSN_COST);
 7522   format %{ "ldrsh  $dst, $mem\t# short" %}
 7523 
 7524   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7525 
 7526   ins_pipe(iload_reg_mem);
 7527 %}
 7528 
 7529 // Load Char (16 bit unsigned)
 7530 instruct loadUS(iRegINoSp dst, memory2 mem)
 7531 %{
 7532   match(Set dst (LoadUS mem));
 7533   predicate(!needs_acquiring_load(n));
 7534 
 7535   ins_cost(4 * INSN_COST);
 7536   format %{ "ldrh  $dst, $mem\t# short" %}
 7537 
 7538   ins_encode(aarch64_enc_ldrh(dst, mem));
 7539 
 7540   ins_pipe(iload_reg_mem);
 7541 %}
 7542 
 7543 // Load Short/Char (16 bit unsigned) into long
 7544 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7545 %{
 7546   match(Set dst (ConvI2L (LoadUS mem)));
 7547   predicate(!needs_acquiring_load(n->in(1)));
 7548 
 7549   ins_cost(4 * INSN_COST);
 7550   format %{ "ldrh  $dst, $mem\t# short" %}
 7551 
 7552   ins_encode(aarch64_enc_ldrh(dst, mem));
 7553 
 7554   ins_pipe(iload_reg_mem);
 7555 %}
 7556 
 7557 // Load Integer (32 bit signed)
 7558 instruct loadI(iRegINoSp dst, memory4 mem)
 7559 %{
 7560   match(Set dst (LoadI mem));
 7561   predicate(!needs_acquiring_load(n));
 7562 
 7563   ins_cost(4 * INSN_COST);
 7564   format %{ "ldrw  $dst, $mem\t# int" %}
 7565 
 7566   ins_encode(aarch64_enc_ldrw(dst, mem));
 7567 
 7568   ins_pipe(iload_reg_mem);
 7569 %}
 7570 
 7571 // Load Integer (32 bit signed) into long
 7572 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7573 %{
 7574   match(Set dst (ConvI2L (LoadI mem)));
 7575   predicate(!needs_acquiring_load(n->in(1)));
 7576 
 7577   ins_cost(4 * INSN_COST);
 7578   format %{ "ldrsw  $dst, $mem\t# int" %}
 7579 
 7580   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7581 
 7582   ins_pipe(iload_reg_mem);
 7583 %}
 7584 
 7585 // Load Integer (32 bit unsigned) into long
 7586 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7587 %{
 7588   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7589   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7590 
 7591   ins_cost(4 * INSN_COST);
 7592   format %{ "ldrw  $dst, $mem\t# int" %}
 7593 
 7594   ins_encode(aarch64_enc_ldrw(dst, mem));
 7595 
 7596   ins_pipe(iload_reg_mem);
 7597 %}
 7598 
 7599 // Load Long (64 bit signed)
 7600 instruct loadL(iRegLNoSp dst, memory8 mem)
 7601 %{
 7602   match(Set dst (LoadL mem));
 7603   predicate(!needs_acquiring_load(n));
 7604 
 7605   ins_cost(4 * INSN_COST);
 7606   format %{ "ldr  $dst, $mem\t# int" %}
 7607 
 7608   ins_encode(aarch64_enc_ldr(dst, mem));
 7609 
 7610   ins_pipe(iload_reg_mem);
 7611 %}
 7612 
 7613 // Load Range
 7614 instruct loadRange(iRegINoSp dst, memory4 mem)
 7615 %{
 7616   match(Set dst (LoadRange mem));
 7617 
 7618   ins_cost(4 * INSN_COST);
 7619   format %{ "ldrw  $dst, $mem\t# range" %}
 7620 
 7621   ins_encode(aarch64_enc_ldrw(dst, mem));
 7622 
 7623   ins_pipe(iload_reg_mem);
 7624 %}
 7625 
 7626 // Load Pointer
 7627 instruct loadP(iRegPNoSp dst, memory8 mem)
 7628 %{
 7629   match(Set dst (LoadP mem));
 7630   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7631 
 7632   ins_cost(4 * INSN_COST);
 7633   format %{ "ldr  $dst, $mem\t# ptr" %}
 7634 
 7635   ins_encode(aarch64_enc_ldr(dst, mem));
 7636 
 7637   ins_pipe(iload_reg_mem);
 7638 %}
 7639 
 7640 // Load Compressed Pointer
 7641 instruct loadN(iRegNNoSp dst, memory4 mem)
 7642 %{
 7643   match(Set dst (LoadN mem));
 7644   predicate(!needs_acquiring_load(n));
 7645 
 7646   ins_cost(4 * INSN_COST);
 7647   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7648 
 7649   ins_encode(aarch64_enc_ldrw(dst, mem));
 7650 
 7651   ins_pipe(iload_reg_mem);
 7652 %}
 7653 
 7654 // Load Klass Pointer
 7655 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7656 %{
 7657   match(Set dst (LoadKlass mem));
 7658   predicate(!needs_acquiring_load(n));
 7659 
 7660   ins_cost(4 * INSN_COST);
 7661   format %{ "ldr  $dst, $mem\t# class" %}
 7662 
 7663   ins_encode(aarch64_enc_ldr(dst, mem));
 7664 
 7665   ins_pipe(iload_reg_mem);
 7666 %}
 7667 
 7668 // Load Narrow Klass Pointer
 7669 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7670 %{
 7671   match(Set dst (LoadNKlass mem));
 7672   predicate(!needs_acquiring_load(n));
 7673 
 7674   ins_cost(4 * INSN_COST);
 7675   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7676 
 7677   ins_encode(aarch64_enc_ldrw(dst, mem));
 7678 
 7679   ins_pipe(iload_reg_mem);
 7680 %}
 7681 
 7682 // Load Float
 7683 instruct loadF(vRegF dst, memory4 mem)
 7684 %{
 7685   match(Set dst (LoadF mem));
 7686   predicate(!needs_acquiring_load(n));
 7687 
 7688   ins_cost(4 * INSN_COST);
 7689   format %{ "ldrs  $dst, $mem\t# float" %}
 7690 
 7691   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7692 
 7693   ins_pipe(pipe_class_memory);
 7694 %}
 7695 
 7696 // Load Double
 7697 instruct loadD(vRegD dst, memory8 mem)
 7698 %{
 7699   match(Set dst (LoadD mem));
 7700   predicate(!needs_acquiring_load(n));
 7701 
 7702   ins_cost(4 * INSN_COST);
 7703   format %{ "ldrd  $dst, $mem\t# double" %}
 7704 
 7705   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7706 
 7707   ins_pipe(pipe_class_memory);
 7708 %}
 7709 
 7710 
 7711 // Load Int Constant
 7712 instruct loadConI(iRegINoSp dst, immI src)
 7713 %{
 7714   match(Set dst src);
 7715 
 7716   ins_cost(INSN_COST);
 7717   format %{ "mov $dst, $src\t# int" %}
 7718 
 7719   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7720 
 7721   ins_pipe(ialu_imm);
 7722 %}
 7723 
 7724 // Load Long Constant
 7725 instruct loadConL(iRegLNoSp dst, immL src)
 7726 %{
 7727   match(Set dst src);
 7728 
 7729   ins_cost(INSN_COST);
 7730   format %{ "mov $dst, $src\t# long" %}
 7731 
 7732   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7733 
 7734   ins_pipe(ialu_imm);
 7735 %}
 7736 
 7737 // Load Pointer Constant
 7738 
 7739 instruct loadConP(iRegPNoSp dst, immP con)
 7740 %{
 7741   match(Set dst con);
 7742 
 7743   ins_cost(INSN_COST * 4);
 7744   format %{
 7745     "mov  $dst, $con\t# ptr"
 7746   %}
 7747 
 7748   ins_encode(aarch64_enc_mov_p(dst, con));
 7749 
 7750   ins_pipe(ialu_imm);
 7751 %}
 7752 
 7753 // Load Null Pointer Constant
 7754 
 7755 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7756 %{
 7757   match(Set dst con);
 7758 
 7759   ins_cost(INSN_COST);
 7760   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7761 
 7762   ins_encode(aarch64_enc_mov_p0(dst, con));
 7763 
 7764   ins_pipe(ialu_imm);
 7765 %}
 7766 
 7767 // Load Pointer Constant One
 7768 
 7769 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7770 %{
 7771   match(Set dst con);
 7772 
 7773   ins_cost(INSN_COST);
 7774   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7775 
 7776   ins_encode(aarch64_enc_mov_p1(dst, con));
 7777 
 7778   ins_pipe(ialu_imm);
 7779 %}
 7780 
 7781 // Load Byte Map Base Constant
 7782 
 7783 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7784 %{
 7785   match(Set dst con);
 7786 
 7787   ins_cost(INSN_COST);
 7788   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7789 
 7790   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7791 
 7792   ins_pipe(ialu_imm);
 7793 %}
 7794 
 7795 // Load Narrow Pointer Constant
 7796 
 7797 instruct loadConN(iRegNNoSp dst, immN con)
 7798 %{
 7799   match(Set dst con);
 7800 
 7801   ins_cost(INSN_COST * 4);
 7802   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7803 
 7804   ins_encode(aarch64_enc_mov_n(dst, con));
 7805 
 7806   ins_pipe(ialu_imm);
 7807 %}
 7808 
 7809 // Load Narrow Null Pointer Constant
 7810 
 7811 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7812 %{
 7813   match(Set dst con);
 7814 
 7815   ins_cost(INSN_COST);
 7816   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7817 
 7818   ins_encode(aarch64_enc_mov_n0(dst, con));
 7819 
 7820   ins_pipe(ialu_imm);
 7821 %}
 7822 
 7823 // Load Narrow Klass Constant
 7824 
 7825 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7826 %{
 7827   match(Set dst con);
 7828 
 7829   ins_cost(INSN_COST);
 7830   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7831 
 7832   ins_encode(aarch64_enc_mov_nk(dst, con));
 7833 
 7834   ins_pipe(ialu_imm);
 7835 %}
 7836 
 7837 // Load Packed Float Constant
 7838 
 7839 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7840   match(Set dst con);
 7841   ins_cost(INSN_COST * 4);
 7842   format %{ "fmovs  $dst, $con"%}
 7843   ins_encode %{
 7844     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7845   %}
 7846 
 7847   ins_pipe(fp_imm_s);
 7848 %}
 7849 
 7850 // Load Float Constant
 7851 
 7852 instruct loadConF(vRegF dst, immF con) %{
 7853   match(Set dst con);
 7854 
 7855   ins_cost(INSN_COST * 4);
 7856 
 7857   format %{
 7858     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7859   %}
 7860 
 7861   ins_encode %{
 7862     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7863   %}
 7864 
 7865   ins_pipe(fp_load_constant_s);
 7866 %}
 7867 
 7868 // Load Packed Double Constant
 7869 
 7870 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7871   match(Set dst con);
 7872   ins_cost(INSN_COST);
 7873   format %{ "fmovd  $dst, $con"%}
 7874   ins_encode %{
 7875     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7876   %}
 7877 
 7878   ins_pipe(fp_imm_d);
 7879 %}
 7880 
 7881 // Load Double Constant
 7882 
 7883 instruct loadConD(vRegD dst, immD con) %{
 7884   match(Set dst con);
 7885 
 7886   ins_cost(INSN_COST * 5);
 7887   format %{
 7888     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7889   %}
 7890 
 7891   ins_encode %{
 7892     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7893   %}
 7894 
 7895   ins_pipe(fp_load_constant_d);
 7896 %}
 7897 
 7898 // Store Instructions
 7899 
 7900 // Store CMS card-mark Immediate
 7901 instruct storeimmCM0(immI0 zero, memory1 mem)
 7902 %{
 7903   match(Set mem (StoreCM mem zero));
 7904 
 7905   ins_cost(INSN_COST);
 7906   format %{ "storestore (elided)\n\t"
 7907             "strb zr, $mem\t# byte" %}
 7908 
 7909   ins_encode(aarch64_enc_strb0(mem));
 7910 
 7911   ins_pipe(istore_mem);
 7912 %}
 7913 
 7914 // Store CMS card-mark Immediate with intervening StoreStore
 7915 // needed when using CMS with no conditional card marking
 7916 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7917 %{
 7918   match(Set mem (StoreCM mem zero));
 7919 
 7920   ins_cost(INSN_COST * 2);
 7921   format %{ "storestore\n\t"
 7922             "dmb ishst"
 7923             "\n\tstrb zr, $mem\t# byte" %}
 7924 
 7925   ins_encode(aarch64_enc_strb0_ordered(mem));
 7926 
 7927   ins_pipe(istore_mem);
 7928 %}
 7929 
 7930 // Store Byte
 7931 instruct storeB(iRegIorL2I src, memory1 mem)
 7932 %{
 7933   match(Set mem (StoreB mem src));
 7934   predicate(!needs_releasing_store(n));
 7935 
 7936   ins_cost(INSN_COST);
 7937   format %{ "strb  $src, $mem\t# byte" %}
 7938 
 7939   ins_encode(aarch64_enc_strb(src, mem));
 7940 
 7941   ins_pipe(istore_reg_mem);
 7942 %}
 7943 
 7944 
 7945 instruct storeimmB0(immI0 zero, memory1 mem)
 7946 %{
 7947   match(Set mem (StoreB mem zero));
 7948   predicate(!needs_releasing_store(n));
 7949 
 7950   ins_cost(INSN_COST);
 7951   format %{ "strb rscractch2, $mem\t# byte" %}
 7952 
 7953   ins_encode(aarch64_enc_strb0(mem));
 7954 
 7955   ins_pipe(istore_mem);
 7956 %}
 7957 
 7958 // Store Char/Short
 7959 instruct storeC(iRegIorL2I src, memory2 mem)
 7960 %{
 7961   match(Set mem (StoreC mem src));
 7962   predicate(!needs_releasing_store(n));
 7963 
 7964   ins_cost(INSN_COST);
 7965   format %{ "strh  $src, $mem\t# short" %}
 7966 
 7967   ins_encode(aarch64_enc_strh(src, mem));
 7968 
 7969   ins_pipe(istore_reg_mem);
 7970 %}
 7971 
 7972 instruct storeimmC0(immI0 zero, memory2 mem)
 7973 %{
 7974   match(Set mem (StoreC mem zero));
 7975   predicate(!needs_releasing_store(n));
 7976 
 7977   ins_cost(INSN_COST);
 7978   format %{ "strh  zr, $mem\t# short" %}
 7979 
 7980   ins_encode(aarch64_enc_strh0(mem));
 7981 
 7982   ins_pipe(istore_mem);
 7983 %}
 7984 
 7985 // Store Integer
 7986 
 7987 instruct storeI(iRegIorL2I src, memory4 mem)
 7988 %{
 7989   match(Set mem(StoreI mem src));
 7990   predicate(!needs_releasing_store(n));
 7991 
 7992   ins_cost(INSN_COST);
 7993   format %{ "strw  $src, $mem\t# int" %}
 7994 
 7995   ins_encode(aarch64_enc_strw(src, mem));
 7996 
 7997   ins_pipe(istore_reg_mem);
 7998 %}
 7999 
 8000 instruct storeimmI0(immI0 zero, memory4 mem)
 8001 %{
 8002   match(Set mem(StoreI mem zero));
 8003   predicate(!needs_releasing_store(n));
 8004 
 8005   ins_cost(INSN_COST);
 8006   format %{ "strw  zr, $mem\t# int" %}
 8007 
 8008   ins_encode(aarch64_enc_strw0(mem));
 8009 
 8010   ins_pipe(istore_mem);
 8011 %}
 8012 
 8013 // Store Long (64 bit signed)
 8014 instruct storeL(iRegL src, memory8 mem)
 8015 %{
 8016   match(Set mem (StoreL mem src));
 8017   predicate(!needs_releasing_store(n));
 8018 
 8019   ins_cost(INSN_COST);
 8020   format %{ "str  $src, $mem\t# int" %}
 8021 
 8022   ins_encode(aarch64_enc_str(src, mem));
 8023 
 8024   ins_pipe(istore_reg_mem);
 8025 %}
 8026 
 8027 // Store Long (64 bit signed)
 8028 instruct storeimmL0(immL0 zero, memory8 mem)
 8029 %{
 8030   match(Set mem (StoreL mem zero));
 8031   predicate(!needs_releasing_store(n));
 8032 
 8033   ins_cost(INSN_COST);
 8034   format %{ "str  zr, $mem\t# int" %}
 8035 
 8036   ins_encode(aarch64_enc_str0(mem));
 8037 
 8038   ins_pipe(istore_mem);
 8039 %}
 8040 
 8041 // Store Pointer
 8042 instruct storeP(iRegP src, memory8 mem)
 8043 %{
 8044   match(Set mem (StoreP mem src));
 8045   predicate(!needs_releasing_store(n));
 8046 
 8047   ins_cost(INSN_COST);
 8048   format %{ "str  $src, $mem\t# ptr" %}
 8049 
 8050   ins_encode(aarch64_enc_str(src, mem));
 8051 
 8052   ins_pipe(istore_reg_mem);
 8053 %}
 8054 
 8055 // Store Pointer
 8056 instruct storeimmP0(immP0 zero, memory8 mem)
 8057 %{
 8058   match(Set mem (StoreP mem zero));
 8059   predicate(!needs_releasing_store(n));
 8060 
 8061   ins_cost(INSN_COST);
 8062   format %{ "str zr, $mem\t# ptr" %}
 8063 
 8064   ins_encode(aarch64_enc_str0(mem));
 8065 
 8066   ins_pipe(istore_mem);
 8067 %}
 8068 
 8069 // Store Compressed Pointer
 8070 instruct storeN(iRegN src, memory4 mem)
 8071 %{
 8072   match(Set mem (StoreN mem src));
 8073   predicate(!needs_releasing_store(n));
 8074 
 8075   ins_cost(INSN_COST);
 8076   format %{ "strw  $src, $mem\t# compressed ptr" %}
 8077 
 8078   ins_encode(aarch64_enc_strw(src, mem));
 8079 
 8080   ins_pipe(istore_reg_mem);
 8081 %}
 8082 
 8083 instruct storeImmN0(immN0 zero, memory4 mem)
 8084 %{
 8085   match(Set mem (StoreN mem zero));
 8086   predicate(!needs_releasing_store(n));
 8087 
 8088   ins_cost(INSN_COST);
 8089   format %{ "strw  zr, $mem\t# compressed ptr" %}
 8090 
 8091   ins_encode(aarch64_enc_strw0(mem));
 8092 
 8093   ins_pipe(istore_mem);
 8094 %}
 8095 
 8096 // Store Float
 8097 instruct storeF(vRegF src, memory4 mem)
 8098 %{
 8099   match(Set mem (StoreF mem src));
 8100   predicate(!needs_releasing_store(n));
 8101 
 8102   ins_cost(INSN_COST);
 8103   format %{ "strs  $src, $mem\t# float" %}
 8104 
 8105   ins_encode( aarch64_enc_strs(src, mem) );
 8106 
 8107   ins_pipe(pipe_class_memory);
 8108 %}
 8109 
 8110 // TODO
 8111 // implement storeImmF0 and storeFImmPacked
 8112 
 8113 // Store Double
 8114 instruct storeD(vRegD src, memory8 mem)
 8115 %{
 8116   match(Set mem (StoreD mem src));
 8117   predicate(!needs_releasing_store(n));
 8118 
 8119   ins_cost(INSN_COST);
 8120   format %{ "strd  $src, $mem\t# double" %}
 8121 
 8122   ins_encode( aarch64_enc_strd(src, mem) );
 8123 
 8124   ins_pipe(pipe_class_memory);
 8125 %}
 8126 
 8127 // Store Compressed Klass Pointer
 8128 instruct storeNKlass(iRegN src, memory4 mem)
 8129 %{
 8130   predicate(!needs_releasing_store(n));
 8131   match(Set mem (StoreNKlass mem src));
 8132 
 8133   ins_cost(INSN_COST);
 8134   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 8135 
 8136   ins_encode(aarch64_enc_strw(src, mem));
 8137 
 8138   ins_pipe(istore_reg_mem);
 8139 %}
 8140 
 8141 // TODO
 8142 // implement storeImmD0 and storeDImmPacked
 8143 
 8144 // prefetch instructions
 8145 // Must be safe to execute with invalid address (cannot fault).
 8146 
 8147 instruct prefetchalloc( memory8 mem ) %{
 8148   match(PrefetchAllocation mem);
 8149 
 8150   ins_cost(INSN_COST);
 8151   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 8152 
 8153   ins_encode( aarch64_enc_prefetchw(mem) );
 8154 
 8155   ins_pipe(iload_prefetch);
 8156 %}
 8157 
 8158 //  ---------------- volatile loads and stores ----------------
 8159 
 8160 // Load Byte (8 bit signed)
 8161 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8162 %{
 8163   match(Set dst (LoadB mem));
 8164 
 8165   ins_cost(VOLATILE_REF_COST);
 8166   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8167 
 8168   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8169 
 8170   ins_pipe(pipe_serial);
 8171 %}
 8172 
 8173 // Load Byte (8 bit signed) into long
 8174 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8175 %{
 8176   match(Set dst (ConvI2L (LoadB mem)));
 8177 
 8178   ins_cost(VOLATILE_REF_COST);
 8179   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8180 
 8181   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8182 
 8183   ins_pipe(pipe_serial);
 8184 %}
 8185 
 8186 // Load Byte (8 bit unsigned)
 8187 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8188 %{
 8189   match(Set dst (LoadUB mem));
 8190 
 8191   ins_cost(VOLATILE_REF_COST);
 8192   format %{ "ldarb  $dst, $mem\t# byte" %}
 8193 
 8194   ins_encode(aarch64_enc_ldarb(dst, mem));
 8195 
 8196   ins_pipe(pipe_serial);
 8197 %}
 8198 
 8199 // Load Byte (8 bit unsigned) into long
 8200 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8201 %{
 8202   match(Set dst (ConvI2L (LoadUB mem)));
 8203 
 8204   ins_cost(VOLATILE_REF_COST);
 8205   format %{ "ldarb  $dst, $mem\t# byte" %}
 8206 
 8207   ins_encode(aarch64_enc_ldarb(dst, mem));
 8208 
 8209   ins_pipe(pipe_serial);
 8210 %}
 8211 
 8212 // Load Short (16 bit signed)
 8213 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8214 %{
 8215   match(Set dst (LoadS mem));
 8216 
 8217   ins_cost(VOLATILE_REF_COST);
 8218   format %{ "ldarshw  $dst, $mem\t# short" %}
 8219 
 8220   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8221 
 8222   ins_pipe(pipe_serial);
 8223 %}
 8224 
 8225 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8226 %{
 8227   match(Set dst (LoadUS mem));
 8228 
 8229   ins_cost(VOLATILE_REF_COST);
 8230   format %{ "ldarhw  $dst, $mem\t# short" %}
 8231 
 8232   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8233 
 8234   ins_pipe(pipe_serial);
 8235 %}
 8236 
 8237 // Load Short/Char (16 bit unsigned) into long
 8238 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8239 %{
 8240   match(Set dst (ConvI2L (LoadUS mem)));
 8241 
 8242   ins_cost(VOLATILE_REF_COST);
 8243   format %{ "ldarh  $dst, $mem\t# short" %}
 8244 
 8245   ins_encode(aarch64_enc_ldarh(dst, mem));
 8246 
 8247   ins_pipe(pipe_serial);
 8248 %}
 8249 
 8250 // Load Short/Char (16 bit signed) into long
 8251 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8252 %{
 8253   match(Set dst (ConvI2L (LoadS mem)));
 8254 
 8255   ins_cost(VOLATILE_REF_COST);
 8256   format %{ "ldarh  $dst, $mem\t# short" %}
 8257 
 8258   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8259 
 8260   ins_pipe(pipe_serial);
 8261 %}
 8262 
 8263 // Load Integer (32 bit signed)
 8264 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8265 %{
 8266   match(Set dst (LoadI mem));
 8267 
 8268   ins_cost(VOLATILE_REF_COST);
 8269   format %{ "ldarw  $dst, $mem\t# int" %}
 8270 
 8271   ins_encode(aarch64_enc_ldarw(dst, mem));
 8272 
 8273   ins_pipe(pipe_serial);
 8274 %}
 8275 
 8276 // Load Integer (32 bit unsigned) into long
 8277 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8278 %{
 8279   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8280 
 8281   ins_cost(VOLATILE_REF_COST);
 8282   format %{ "ldarw  $dst, $mem\t# int" %}
 8283 
 8284   ins_encode(aarch64_enc_ldarw(dst, mem));
 8285 
 8286   ins_pipe(pipe_serial);
 8287 %}
 8288 
 8289 // Load Long (64 bit signed)
 8290 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8291 %{
 8292   match(Set dst (LoadL mem));
 8293 
 8294   ins_cost(VOLATILE_REF_COST);
 8295   format %{ "ldar  $dst, $mem\t# int" %}
 8296 
 8297   ins_encode(aarch64_enc_ldar(dst, mem));
 8298 
 8299   ins_pipe(pipe_serial);
 8300 %}
 8301 
 8302 // Load Pointer
 8303 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8304 %{
 8305   match(Set dst (LoadP mem));
 8306   predicate(n->as_Load()->barrier_data() == 0);
 8307 
 8308   ins_cost(VOLATILE_REF_COST);
 8309   format %{ "ldar  $dst, $mem\t# ptr" %}
 8310 
 8311   ins_encode(aarch64_enc_ldar(dst, mem));
 8312 
 8313   ins_pipe(pipe_serial);
 8314 %}
 8315 
 8316 // Load Compressed Pointer
 8317 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8318 %{
 8319   match(Set dst (LoadN mem));
 8320 
 8321   ins_cost(VOLATILE_REF_COST);
 8322   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8323 
 8324   ins_encode(aarch64_enc_ldarw(dst, mem));
 8325 
 8326   ins_pipe(pipe_serial);
 8327 %}
 8328 
 8329 // Load Float
 8330 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8331 %{
 8332   match(Set dst (LoadF mem));
 8333 
 8334   ins_cost(VOLATILE_REF_COST);
 8335   format %{ "ldars  $dst, $mem\t# float" %}
 8336 
 8337   ins_encode( aarch64_enc_fldars(dst, mem) );
 8338 
 8339   ins_pipe(pipe_serial);
 8340 %}
 8341 
 8342 // Load Double
 8343 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8344 %{
 8345   match(Set dst (LoadD mem));
 8346 
 8347   ins_cost(VOLATILE_REF_COST);
 8348   format %{ "ldard  $dst, $mem\t# double" %}
 8349 
 8350   ins_encode( aarch64_enc_fldard(dst, mem) );
 8351 
 8352   ins_pipe(pipe_serial);
 8353 %}
 8354 
 8355 // Store Byte
 8356 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8357 %{
 8358   match(Set mem (StoreB mem src));
 8359 
 8360   ins_cost(VOLATILE_REF_COST);
 8361   format %{ "stlrb  $src, $mem\t# byte" %}
 8362 
 8363   ins_encode(aarch64_enc_stlrb(src, mem));
 8364 
 8365   ins_pipe(pipe_class_memory);
 8366 %}
 8367 
 8368 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8369 %{
 8370   match(Set mem (StoreB mem zero));
 8371 
 8372   ins_cost(VOLATILE_REF_COST);
 8373   format %{ "stlrb  zr, $mem\t# byte" %}
 8374 
 8375   ins_encode(aarch64_enc_stlrb0(mem));
 8376 
 8377   ins_pipe(pipe_class_memory);
 8378 %}
 8379 
 8380 // Store Char/Short
 8381 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8382 %{
 8383   match(Set mem (StoreC mem src));
 8384 
 8385   ins_cost(VOLATILE_REF_COST);
 8386   format %{ "stlrh  $src, $mem\t# short" %}
 8387 
 8388   ins_encode(aarch64_enc_stlrh(src, mem));
 8389 
 8390   ins_pipe(pipe_class_memory);
 8391 %}
 8392 
 8393 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8394 %{
 8395   match(Set mem (StoreC mem zero));
 8396 
 8397   ins_cost(VOLATILE_REF_COST);
 8398   format %{ "stlrh  zr, $mem\t# short" %}
 8399 
 8400   ins_encode(aarch64_enc_stlrh0(mem));
 8401 
 8402   ins_pipe(pipe_class_memory);
 8403 %}
 8404 
 8405 // Store Integer
 8406 
 8407 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8408 %{
 8409   match(Set mem(StoreI mem src));
 8410 
 8411   ins_cost(VOLATILE_REF_COST);
 8412   format %{ "stlrw  $src, $mem\t# int" %}
 8413 
 8414   ins_encode(aarch64_enc_stlrw(src, mem));
 8415 
 8416   ins_pipe(pipe_class_memory);
 8417 %}
 8418 
 8419 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8420 %{
 8421   match(Set mem(StoreI mem zero));
 8422 
 8423   ins_cost(VOLATILE_REF_COST);
 8424   format %{ "stlrw  zr, $mem\t# int" %}
 8425 
 8426   ins_encode(aarch64_enc_stlrw0(mem));
 8427 
 8428   ins_pipe(pipe_class_memory);
 8429 %}
 8430 
 8431 // Store Long (64 bit signed)
 8432 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8433 %{
 8434   match(Set mem (StoreL mem src));
 8435 
 8436   ins_cost(VOLATILE_REF_COST);
 8437   format %{ "stlr  $src, $mem\t# int" %}
 8438 
 8439   ins_encode(aarch64_enc_stlr(src, mem));
 8440 
 8441   ins_pipe(pipe_class_memory);
 8442 %}
 8443 
 8444 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8445 %{
 8446   match(Set mem (StoreL mem zero));
 8447 
 8448   ins_cost(VOLATILE_REF_COST);
 8449   format %{ "stlr  zr, $mem\t# int" %}
 8450 
 8451   ins_encode(aarch64_enc_stlr0(mem));
 8452 
 8453   ins_pipe(pipe_class_memory);
 8454 %}
 8455 
 8456 // Store Pointer
 8457 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8458 %{
 8459   match(Set mem (StoreP mem src));
 8460 
 8461   ins_cost(VOLATILE_REF_COST);
 8462   format %{ "stlr  $src, $mem\t# ptr" %}
 8463 
 8464   ins_encode(aarch64_enc_stlr(src, mem));
 8465 
 8466   ins_pipe(pipe_class_memory);
 8467 %}
 8468 
 8469 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8470 %{
 8471   match(Set mem (StoreP mem zero));
 8472 
 8473   ins_cost(VOLATILE_REF_COST);
 8474   format %{ "stlr  zr, $mem\t# ptr" %}
 8475 
 8476   ins_encode(aarch64_enc_stlr0(mem));
 8477 
 8478   ins_pipe(pipe_class_memory);
 8479 %}
 8480 
 8481 // Store Compressed Pointer
 8482 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8483 %{
 8484   match(Set mem (StoreN mem src));
 8485 
 8486   ins_cost(VOLATILE_REF_COST);
 8487   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8488 
 8489   ins_encode(aarch64_enc_stlrw(src, mem));
 8490 
 8491   ins_pipe(pipe_class_memory);
 8492 %}
 8493 
 8494 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8495 %{
 8496   match(Set mem (StoreN mem zero));
 8497 
 8498   ins_cost(VOLATILE_REF_COST);
 8499   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8500 
 8501   ins_encode(aarch64_enc_stlrw0(mem));
 8502 
 8503   ins_pipe(pipe_class_memory);
 8504 %}
 8505 
 8506 // Store Float
 8507 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8508 %{
 8509   match(Set mem (StoreF mem src));
 8510 
 8511   ins_cost(VOLATILE_REF_COST);
 8512   format %{ "stlrs  $src, $mem\t# float" %}
 8513 
 8514   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8515 
 8516   ins_pipe(pipe_class_memory);
 8517 %}
 8518 
 8519 // TODO
 8520 // implement storeImmF0 and storeFImmPacked
 8521 
 8522 // Store Double
 8523 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8524 %{
 8525   match(Set mem (StoreD mem src));
 8526 
 8527   ins_cost(VOLATILE_REF_COST);
 8528   format %{ "stlrd  $src, $mem\t# double" %}
 8529 
 8530   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8531 
 8532   ins_pipe(pipe_class_memory);
 8533 %}
 8534 
 8535 //  ---------------- end of volatile loads and stores ----------------
 8536 
 8537 instruct cacheWB(indirect addr)
 8538 %{
 8539   predicate(VM_Version::supports_data_cache_line_flush());
 8540   match(CacheWB addr);
 8541 
 8542   ins_cost(100);
 8543   format %{"cache wb $addr" %}
 8544   ins_encode %{
 8545     assert($addr->index_position() < 0, "should be");
 8546     assert($addr$$disp == 0, "should be");
 8547     __ cache_wb(Address($addr$$base$$Register, 0));
 8548   %}
 8549   ins_pipe(pipe_slow); // XXX
 8550 %}
 8551 
 8552 instruct cacheWBPreSync()
 8553 %{
 8554   predicate(VM_Version::supports_data_cache_line_flush());
 8555   match(CacheWBPreSync);
 8556 
 8557   ins_cost(100);
 8558   format %{"cache wb presync" %}
 8559   ins_encode %{
 8560     __ cache_wbsync(true);
 8561   %}
 8562   ins_pipe(pipe_slow); // XXX
 8563 %}
 8564 
 8565 instruct cacheWBPostSync()
 8566 %{
 8567   predicate(VM_Version::supports_data_cache_line_flush());
 8568   match(CacheWBPostSync);
 8569 
 8570   ins_cost(100);
 8571   format %{"cache wb postsync" %}
 8572   ins_encode %{
 8573     __ cache_wbsync(false);
 8574   %}
 8575   ins_pipe(pipe_slow); // XXX
 8576 %}
 8577 
 8578 // ============================================================================
 8579 // BSWAP Instructions
 8580 
 8581 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8582   match(Set dst (ReverseBytesI src));
 8583 
 8584   ins_cost(INSN_COST);
 8585   format %{ "revw  $dst, $src" %}
 8586 
 8587   ins_encode %{
 8588     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8589   %}
 8590 
 8591   ins_pipe(ialu_reg);
 8592 %}
 8593 
 8594 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8595   match(Set dst (ReverseBytesL src));
 8596 
 8597   ins_cost(INSN_COST);
 8598   format %{ "rev  $dst, $src" %}
 8599 
 8600   ins_encode %{
 8601     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8602   %}
 8603 
 8604   ins_pipe(ialu_reg);
 8605 %}
 8606 
 8607 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8608   match(Set dst (ReverseBytesUS src));
 8609 
 8610   ins_cost(INSN_COST);
 8611   format %{ "rev16w  $dst, $src" %}
 8612 
 8613   ins_encode %{
 8614     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8615   %}
 8616 
 8617   ins_pipe(ialu_reg);
 8618 %}
 8619 
 8620 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8621   match(Set dst (ReverseBytesS src));
 8622 
 8623   ins_cost(INSN_COST);
 8624   format %{ "rev16w  $dst, $src\n\t"
 8625             "sbfmw $dst, $dst, #0, #15" %}
 8626 
 8627   ins_encode %{
 8628     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8629     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8630   %}
 8631 
 8632   ins_pipe(ialu_reg);
 8633 %}
 8634 
 8635 // ============================================================================
 8636 // Zero Count Instructions
 8637 
 8638 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8639   match(Set dst (CountLeadingZerosI src));
 8640 
 8641   ins_cost(INSN_COST);
 8642   format %{ "clzw  $dst, $src" %}
 8643   ins_encode %{
 8644     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8645   %}
 8646 
 8647   ins_pipe(ialu_reg);
 8648 %}
 8649 
 8650 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8651   match(Set dst (CountLeadingZerosL src));
 8652 
 8653   ins_cost(INSN_COST);
 8654   format %{ "clz   $dst, $src" %}
 8655   ins_encode %{
 8656     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8657   %}
 8658 
 8659   ins_pipe(ialu_reg);
 8660 %}
 8661 
 8662 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8663   match(Set dst (CountTrailingZerosI src));
 8664 
 8665   ins_cost(INSN_COST * 2);
 8666   format %{ "rbitw  $dst, $src\n\t"
 8667             "clzw   $dst, $dst" %}
 8668   ins_encode %{
 8669     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8670     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8671   %}
 8672 
 8673   ins_pipe(ialu_reg);
 8674 %}
 8675 
 8676 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8677   match(Set dst (CountTrailingZerosL src));
 8678 
 8679   ins_cost(INSN_COST * 2);
 8680   format %{ "rbit   $dst, $src\n\t"
 8681             "clz    $dst, $dst" %}
 8682   ins_encode %{
 8683     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8684     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8685   %}
 8686 
 8687   ins_pipe(ialu_reg);
 8688 %}
 8689 
 8690 //---------- Population Count Instructions -------------------------------------
 8691 //
 8692 
 8693 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8694   predicate(UsePopCountInstruction);
 8695   match(Set dst (PopCountI src));
 8696   effect(TEMP tmp);
 8697   ins_cost(INSN_COST * 13);
 8698 
 8699   format %{ "movw   $src, $src\n\t"
 8700             "mov    $tmp, $src\t# vector (1D)\n\t"
 8701             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8702             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8703             "mov    $dst, $tmp\t# vector (1D)" %}
 8704   ins_encode %{
 8705     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8706     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8707     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8708     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8709     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8710   %}
 8711 
 8712   ins_pipe(pipe_class_default);
 8713 %}
 8714 
 8715 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8716   predicate(UsePopCountInstruction);
 8717   match(Set dst (PopCountI (LoadI mem)));
 8718   effect(TEMP tmp);
 8719   ins_cost(INSN_COST * 13);
 8720 
 8721   format %{ "ldrs   $tmp, $mem\n\t"
 8722             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8723             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8724             "mov    $dst, $tmp\t# vector (1D)" %}
 8725   ins_encode %{
 8726     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8727     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8728               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8729     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8730     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8731     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8732   %}
 8733 
 8734   ins_pipe(pipe_class_default);
 8735 %}
 8736 
 8737 // Note: Long.bitCount(long) returns an int.
 8738 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8739   predicate(UsePopCountInstruction);
 8740   match(Set dst (PopCountL src));
 8741   effect(TEMP tmp);
 8742   ins_cost(INSN_COST * 13);
 8743 
 8744   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8745             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8746             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8747             "mov    $dst, $tmp\t# vector (1D)" %}
 8748   ins_encode %{
 8749     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8750     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8751     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8752     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8753   %}
 8754 
 8755   ins_pipe(pipe_class_default);
 8756 %}
 8757 
 8758 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8759   predicate(UsePopCountInstruction);
 8760   match(Set dst (PopCountL (LoadL mem)));
 8761   effect(TEMP tmp);
 8762   ins_cost(INSN_COST * 13);
 8763 
 8764   format %{ "ldrd   $tmp, $mem\n\t"
 8765             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8766             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8767             "mov    $dst, $tmp\t# vector (1D)" %}
 8768   ins_encode %{
 8769     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8770     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8771               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8772     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8773     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8774     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8775   %}
 8776 
 8777   ins_pipe(pipe_class_default);
 8778 %}
 8779 
 8780 // ============================================================================
 8781 // MemBar Instruction
 8782 
 8783 instruct load_fence() %{
 8784   match(LoadFence);
 8785   ins_cost(VOLATILE_REF_COST);
 8786 
 8787   format %{ "load_fence" %}
 8788 
 8789   ins_encode %{
 8790     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8791   %}
 8792   ins_pipe(pipe_serial);
 8793 %}
 8794 
 8795 instruct unnecessary_membar_acquire() %{
 8796   predicate(unnecessary_acquire(n));
 8797   match(MemBarAcquire);
 8798   ins_cost(0);
 8799 
 8800   format %{ "membar_acquire (elided)" %}
 8801 
 8802   ins_encode %{
 8803     __ block_comment("membar_acquire (elided)");
 8804   %}
 8805 
 8806   ins_pipe(pipe_class_empty);
 8807 %}
 8808 
 8809 instruct membar_acquire() %{
 8810   match(MemBarAcquire);
 8811   ins_cost(VOLATILE_REF_COST);
 8812 
 8813   format %{ "membar_acquire\n\t"
 8814             "dmb ish" %}
 8815 
 8816   ins_encode %{
 8817     __ block_comment("membar_acquire");
 8818     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8819   %}
 8820 
 8821   ins_pipe(pipe_serial);
 8822 %}
 8823 
 8824 
 8825 instruct membar_acquire_lock() %{
 8826   match(MemBarAcquireLock);
 8827   ins_cost(VOLATILE_REF_COST);
 8828 
 8829   format %{ "membar_acquire_lock (elided)" %}
 8830 
 8831   ins_encode %{
 8832     __ block_comment("membar_acquire_lock (elided)");
 8833   %}
 8834 
 8835   ins_pipe(pipe_serial);
 8836 %}
 8837 
 8838 instruct store_fence() %{
 8839   match(StoreFence);
 8840   ins_cost(VOLATILE_REF_COST);
 8841 
 8842   format %{ "store_fence" %}
 8843 
 8844   ins_encode %{
 8845     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8846   %}
 8847   ins_pipe(pipe_serial);
 8848 %}
 8849 
 8850 instruct unnecessary_membar_release() %{
 8851   predicate(unnecessary_release(n));
 8852   match(MemBarRelease);
 8853   ins_cost(0);
 8854 
 8855   format %{ "membar_release (elided)" %}
 8856 
 8857   ins_encode %{
 8858     __ block_comment("membar_release (elided)");
 8859   %}
 8860   ins_pipe(pipe_serial);
 8861 %}
 8862 
 8863 instruct membar_release() %{
 8864   match(MemBarRelease);
 8865   ins_cost(VOLATILE_REF_COST);
 8866 
 8867   format %{ "membar_release\n\t"
 8868             "dmb ish" %}
 8869 
 8870   ins_encode %{
 8871     __ block_comment("membar_release");
 8872     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8873   %}
 8874   ins_pipe(pipe_serial);
 8875 %}
 8876 
 8877 instruct membar_storestore() %{
 8878   match(MemBarStoreStore);
 8879   match(StoreStoreFence);
 8880   ins_cost(VOLATILE_REF_COST);
 8881 
 8882   format %{ "MEMBAR-store-store" %}
 8883 
 8884   ins_encode %{
 8885     __ membar(Assembler::StoreStore);
 8886   %}
 8887   ins_pipe(pipe_serial);
 8888 %}
 8889 
 8890 instruct membar_release_lock() %{
 8891   match(MemBarReleaseLock);
 8892   ins_cost(VOLATILE_REF_COST);
 8893 
 8894   format %{ "membar_release_lock (elided)" %}
 8895 
 8896   ins_encode %{
 8897     __ block_comment("membar_release_lock (elided)");
 8898   %}
 8899 
 8900   ins_pipe(pipe_serial);
 8901 %}
 8902 
 8903 instruct unnecessary_membar_volatile() %{
 8904   predicate(unnecessary_volatile(n));
 8905   match(MemBarVolatile);
 8906   ins_cost(0);
 8907 
 8908   format %{ "membar_volatile (elided)" %}
 8909 
 8910   ins_encode %{
 8911     __ block_comment("membar_volatile (elided)");
 8912   %}
 8913 
 8914   ins_pipe(pipe_serial);
 8915 %}
 8916 
 8917 instruct membar_volatile() %{
 8918   match(MemBarVolatile);
 8919   ins_cost(VOLATILE_REF_COST*100);
 8920 
 8921   format %{ "membar_volatile\n\t"
 8922              "dmb ish"%}
 8923 
 8924   ins_encode %{
 8925     __ block_comment("membar_volatile");
 8926     __ membar(Assembler::StoreLoad);
 8927   %}
 8928 
 8929   ins_pipe(pipe_serial);
 8930 %}
 8931 
 8932 // ============================================================================
 8933 // Cast/Convert Instructions
 8934 
 8935 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8936   match(Set dst (CastX2P src));
 8937 
 8938   ins_cost(INSN_COST);
 8939   format %{ "mov $dst, $src\t# long -> ptr" %}
 8940 
 8941   ins_encode %{
 8942     if ($dst$$reg != $src$$reg) {
 8943       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8944     }
 8945   %}
 8946 
 8947   ins_pipe(ialu_reg);
 8948 %}
 8949 
 8950 instruct castN2X(iRegLNoSp dst, iRegN src) %{
 8951   match(Set dst (CastP2X src));
 8952 
 8953   ins_cost(INSN_COST);
 8954   format %{ "mov $dst, $src\t# ptr -> long" %}
 8955 
 8956   ins_encode %{
 8957     if ($dst$$reg != $src$$reg) {
 8958       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8959     }
 8960   %}
 8961 
 8962   ins_pipe(ialu_reg);
 8963 %}
 8964 
 8965 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8966   match(Set dst (CastP2X src));
 8967 
 8968   ins_cost(INSN_COST);
 8969   format %{ "mov $dst, $src\t# ptr -> long" %}
 8970 
 8971   ins_encode %{
 8972     if ($dst$$reg != $src$$reg) {
 8973       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8974     }
 8975   %}
 8976 
 8977   ins_pipe(ialu_reg);
 8978 %}
 8979 
 8980 // Convert oop into int for vectors alignment masking
 8981 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8982   match(Set dst (ConvL2I (CastP2X src)));
 8983 
 8984   ins_cost(INSN_COST);
 8985   format %{ "movw $dst, $src\t# ptr -> int" %}
 8986   ins_encode %{
 8987     __ movw($dst$$Register, $src$$Register);
 8988   %}
 8989 
 8990   ins_pipe(ialu_reg);
 8991 %}
 8992 
 8993 // Convert compressed oop into int for vectors alignment masking
 8994 // in case of 32bit oops (heap < 4Gb).
 8995 instruct convN2I(iRegINoSp dst, iRegN src)
 8996 %{
 8997   predicate(CompressedOops::shift() == 0);
 8998   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8999 
 9000   ins_cost(INSN_COST);
 9001   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 9002   ins_encode %{
 9003     __ movw($dst$$Register, $src$$Register);
 9004   %}
 9005 
 9006   ins_pipe(ialu_reg);
 9007 %}
 9008 
 9009 
 9010 // Convert oop pointer into compressed form
 9011 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 9012   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 9013   match(Set dst (EncodeP src));
 9014   effect(KILL cr);
 9015   ins_cost(INSN_COST * 3);
 9016   format %{ "encode_heap_oop $dst, $src" %}
 9017   ins_encode %{
 9018     Register s = $src$$Register;
 9019     Register d = $dst$$Register;
 9020     __ encode_heap_oop(d, s);
 9021   %}
 9022   ins_pipe(ialu_reg);
 9023 %}
 9024 
 9025 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 9026   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 9027   match(Set dst (EncodeP src));
 9028   ins_cost(INSN_COST * 3);
 9029   format %{ "encode_heap_oop_not_null $dst, $src" %}
 9030   ins_encode %{
 9031     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 9032   %}
 9033   ins_pipe(ialu_reg);
 9034 %}
 9035 
 9036 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 9037   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 9038             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 9039   match(Set dst (DecodeN src));
 9040   ins_cost(INSN_COST * 3);
 9041   format %{ "decode_heap_oop $dst, $src" %}
 9042   ins_encode %{
 9043     Register s = $src$$Register;
 9044     Register d = $dst$$Register;
 9045     __ decode_heap_oop(d, s);
 9046   %}
 9047   ins_pipe(ialu_reg);
 9048 %}
 9049 
 9050 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 9051   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 9052             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 9053   match(Set dst (DecodeN src));
 9054   ins_cost(INSN_COST * 3);
 9055   format %{ "decode_heap_oop_not_null $dst, $src" %}
 9056   ins_encode %{
 9057     Register s = $src$$Register;
 9058     Register d = $dst$$Register;
 9059     __ decode_heap_oop_not_null(d, s);
 9060   %}
 9061   ins_pipe(ialu_reg);
 9062 %}
 9063 
 9064 // n.b. AArch64 implementations of encode_klass_not_null and
 9065 // decode_klass_not_null do not modify the flags register so, unlike
 9066 // Intel, we don't kill CR as a side effect here
 9067 
 9068 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 9069   match(Set dst (EncodePKlass src));
 9070 
 9071   ins_cost(INSN_COST * 3);
 9072   format %{ "encode_klass_not_null $dst,$src" %}
 9073 
 9074   ins_encode %{
 9075     Register src_reg = as_Register($src$$reg);
 9076     Register dst_reg = as_Register($dst$$reg);
 9077     __ encode_klass_not_null(dst_reg, src_reg);
 9078   %}
 9079 
 9080    ins_pipe(ialu_reg);
 9081 %}
 9082 
 9083 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 9084   match(Set dst (DecodeNKlass src));
 9085 
 9086   ins_cost(INSN_COST * 3);
 9087   format %{ "decode_klass_not_null $dst,$src" %}
 9088 
 9089   ins_encode %{
 9090     Register src_reg = as_Register($src$$reg);
 9091     Register dst_reg = as_Register($dst$$reg);
 9092     if (dst_reg != src_reg) {
 9093       __ decode_klass_not_null(dst_reg, src_reg);
 9094     } else {
 9095       __ decode_klass_not_null(dst_reg);
 9096     }
 9097   %}
 9098 
 9099    ins_pipe(ialu_reg);
 9100 %}
 9101 
 9102 instruct checkCastPP(iRegPNoSp dst)
 9103 %{
 9104   match(Set dst (CheckCastPP dst));
 9105 
 9106   size(0);
 9107   format %{ "# checkcastPP of $dst" %}
 9108   ins_encode(/* empty encoding */);
 9109   ins_pipe(pipe_class_empty);
 9110 %}
 9111 
 9112 instruct castPP(iRegPNoSp dst)
 9113 %{
 9114   match(Set dst (CastPP dst));
 9115 
 9116   size(0);
 9117   format %{ "# castPP of $dst" %}
 9118   ins_encode(/* empty encoding */);
 9119   ins_pipe(pipe_class_empty);
 9120 %}
 9121 
 9122 instruct castII(iRegI dst)
 9123 %{
 9124   match(Set dst (CastII dst));
 9125 
 9126   size(0);
 9127   format %{ "# castII of $dst" %}
 9128   ins_encode(/* empty encoding */);
 9129   ins_cost(0);
 9130   ins_pipe(pipe_class_empty);
 9131 %}
 9132 
 9133 instruct castLL(iRegL dst)
 9134 %{
 9135   match(Set dst (CastLL dst));
 9136 
 9137   size(0);
 9138   format %{ "# castLL of $dst" %}
 9139   ins_encode(/* empty encoding */);
 9140   ins_cost(0);
 9141   ins_pipe(pipe_class_empty);
 9142 %}
 9143 
 9144 instruct castFF(vRegF dst)
 9145 %{
 9146   match(Set dst (CastFF dst));
 9147 
 9148   size(0);
 9149   format %{ "# castFF of $dst" %}
 9150   ins_encode(/* empty encoding */);
 9151   ins_cost(0);
 9152   ins_pipe(pipe_class_empty);
 9153 %}
 9154 
 9155 instruct castDD(vRegD dst)
 9156 %{
 9157   match(Set dst (CastDD dst));
 9158 
 9159   size(0);
 9160   format %{ "# castDD of $dst" %}
 9161   ins_encode(/* empty encoding */);
 9162   ins_cost(0);
 9163   ins_pipe(pipe_class_empty);
 9164 %}
 9165 
 9166 instruct castVVD(vecD dst)
 9167 %{
 9168   match(Set dst (CastVV dst));
 9169 
 9170   size(0);
 9171   format %{ "# castVV of $dst" %}
 9172   ins_encode(/* empty encoding */);
 9173   ins_cost(0);
 9174   ins_pipe(pipe_class_empty);
 9175 %}
 9176 
 9177 instruct castVVX(vecX dst)
 9178 %{
 9179   match(Set dst (CastVV dst));
 9180 
 9181   size(0);
 9182   format %{ "# castVV of $dst" %}
 9183   ins_encode(/* empty encoding */);
 9184   ins_cost(0);
 9185   ins_pipe(pipe_class_empty);
 9186 %}
 9187 
 9188 instruct castVV(vReg dst)
 9189 %{
 9190   match(Set dst (CastVV dst));
 9191 
 9192   size(0);
 9193   format %{ "# castVV of $dst" %}
 9194   ins_encode(/* empty encoding */);
 9195   ins_cost(0);
 9196   ins_pipe(pipe_class_empty);
 9197 %}
 9198 
 9199 instruct castVVMask(pRegGov dst)
 9200 %{
 9201   match(Set dst (CastVV dst));
 9202 
 9203   size(0);
 9204   format %{ "# castVV of $dst" %}
 9205   ins_encode(/* empty encoding */);
 9206   ins_cost(0);
 9207   ins_pipe(pipe_class_empty);
 9208 %}
 9209 
 9210 // ============================================================================
 9211 // Atomic operation instructions
 9212 //
 9213 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9214 // Store{PIL}Conditional instructions using a normal load for the
 9215 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9216 //
 9217 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9218 // pair to lock object allocations from Eden space when not using
 9219 // TLABs.
 9220 //
 9221 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9222 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9223 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9224 // only for 64-bit.
 9225 //
 9226 // We implement LoadPLocked and StorePLocked instructions using,
 9227 // respectively the AArch64 hw load-exclusive and store-conditional
 9228 // instructions. Whereas we must implement each of
 9229 // Store{IL}Conditional using a CAS which employs a pair of
 9230 // instructions comprising a load-exclusive followed by a
 9231 // store-conditional.
 9232 
 9233 
 9234 // Locked-load (linked load) of the current heap-top
 9235 // used when updating the eden heap top
 9236 // implemented using ldaxr on AArch64
 9237 
 9238 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9239 %{
 9240   match(Set dst (LoadPLocked mem));
 9241 
 9242   ins_cost(VOLATILE_REF_COST);
 9243 
 9244   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9245 
 9246   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9247 
 9248   ins_pipe(pipe_serial);
 9249 %}
 9250 
 9251 // Conditional-store of the updated heap-top.
 9252 // Used during allocation of the shared heap.
 9253 // Sets flag (EQ) on success.
 9254 // implemented using stlxr on AArch64.
 9255 
 9256 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9257 %{
 9258   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9259 
 9260   ins_cost(VOLATILE_REF_COST);
 9261 
 9262  // TODO
 9263  // do we need to do a store-conditional release or can we just use a
 9264  // plain store-conditional?
 9265 
 9266   format %{
 9267     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9268     "cmpw rscratch1, zr\t# EQ on successful write"
 9269   %}
 9270 
 9271   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9272 
 9273   ins_pipe(pipe_serial);
 9274 %}
 9275 
 9276 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9277 %{
 9278   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9279 
 9280   ins_cost(VOLATILE_REF_COST);
 9281 
 9282   format %{
 9283     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9284     "cmpw rscratch1, zr\t# EQ on successful write"
 9285   %}
 9286 
 9287   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9288 
 9289   ins_pipe(pipe_slow);
 9290 %}
 9291 
 9292 // storeIConditional also has acquire semantics, for no better reason
 9293 // than matching storeLConditional.  At the time of writing this
 9294 // comment storeIConditional was not used anywhere by AArch64.
 9295 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9296 %{
 9297   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9298 
 9299   ins_cost(VOLATILE_REF_COST);
 9300 
 9301   format %{
 9302     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9303     "cmpw rscratch1, zr\t# EQ on successful write"
 9304   %}
 9305 
 9306   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9307 
 9308   ins_pipe(pipe_slow);
 9309 %}
 9310 
 9311 // standard CompareAndSwapX when we are using barriers
 9312 // these have higher priority than the rules selected by a predicate
 9313 
 9314 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9315 // can't match them
 9316 
 9317 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9318 
 9319   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9320   ins_cost(2 * VOLATILE_REF_COST);
 9321 
 9322   effect(KILL cr);
 9323 
 9324   format %{
 9325     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9326     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9327   %}
 9328 
 9329   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9330             aarch64_enc_cset_eq(res));
 9331 
 9332   ins_pipe(pipe_slow);
 9333 %}
 9334 
 9335 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9336 
 9337   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9338   ins_cost(2 * VOLATILE_REF_COST);
 9339 
 9340   effect(KILL cr);
 9341 
 9342   format %{
 9343     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9344     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9345   %}
 9346 
 9347   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9348             aarch64_enc_cset_eq(res));
 9349 
 9350   ins_pipe(pipe_slow);
 9351 %}
 9352 
 9353 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9354 
 9355   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9356   ins_cost(2 * VOLATILE_REF_COST);
 9357 
 9358   effect(KILL cr);
 9359 
 9360  format %{
 9361     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9362     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9363  %}
 9364 
 9365  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9366             aarch64_enc_cset_eq(res));
 9367 
 9368   ins_pipe(pipe_slow);
 9369 %}
 9370 
 9371 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9372 
 9373   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9374   ins_cost(2 * VOLATILE_REF_COST);
 9375 
 9376   effect(KILL cr);
 9377 
 9378  format %{
 9379     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9380     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9381  %}
 9382 
 9383  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9384             aarch64_enc_cset_eq(res));
 9385 
 9386   ins_pipe(pipe_slow);
 9387 %}
 9388 
 9389 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9390 
 9391   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9392   predicate(n->as_LoadStore()->barrier_data() == 0);
 9393   ins_cost(2 * VOLATILE_REF_COST);
 9394 
 9395   effect(KILL cr);
 9396 
 9397  format %{
 9398     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9399     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9400  %}
 9401 
 9402  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9403             aarch64_enc_cset_eq(res));
 9404 
 9405   ins_pipe(pipe_slow);
 9406 %}
 9407 
 9408 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9409 
 9410   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9411   ins_cost(2 * VOLATILE_REF_COST);
 9412 
 9413   effect(KILL cr);
 9414 
 9415  format %{
 9416     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9417     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9418  %}
 9419 
 9420  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9421             aarch64_enc_cset_eq(res));
 9422 
 9423   ins_pipe(pipe_slow);
 9424 %}
 9425 
 9426 // alternative CompareAndSwapX when we are eliding barriers
 9427 
 9428 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9429 
 9430   predicate(needs_acquiring_load_exclusive(n));
 9431   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9432   ins_cost(VOLATILE_REF_COST);
 9433 
 9434   effect(KILL cr);
 9435 
 9436   format %{
 9437     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9438     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9439   %}
 9440 
 9441   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9442             aarch64_enc_cset_eq(res));
 9443 
 9444   ins_pipe(pipe_slow);
 9445 %}
 9446 
 9447 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9448 
 9449   predicate(needs_acquiring_load_exclusive(n));
 9450   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9451   ins_cost(VOLATILE_REF_COST);
 9452 
 9453   effect(KILL cr);
 9454 
 9455   format %{
 9456     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9457     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9458   %}
 9459 
 9460   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9461             aarch64_enc_cset_eq(res));
 9462 
 9463   ins_pipe(pipe_slow);
 9464 %}
 9465 
 9466 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9467 
 9468   predicate(needs_acquiring_load_exclusive(n));
 9469   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9470   ins_cost(VOLATILE_REF_COST);
 9471 
 9472   effect(KILL cr);
 9473 
 9474  format %{
 9475     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9476     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9477  %}
 9478 
 9479  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9480             aarch64_enc_cset_eq(res));
 9481 
 9482   ins_pipe(pipe_slow);
 9483 %}
 9484 
 9485 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9486 
 9487   predicate(needs_acquiring_load_exclusive(n));
 9488   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9489   ins_cost(VOLATILE_REF_COST);
 9490 
 9491   effect(KILL cr);
 9492 
 9493  format %{
 9494     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9495     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9496  %}
 9497 
 9498  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9499             aarch64_enc_cset_eq(res));
 9500 
 9501   ins_pipe(pipe_slow);
 9502 %}
 9503 
 9504 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9505 
 9506   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9507   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9508   ins_cost(VOLATILE_REF_COST);
 9509 
 9510   effect(KILL cr);
 9511 
 9512  format %{
 9513     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9514     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9515  %}
 9516 
 9517  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9518             aarch64_enc_cset_eq(res));
 9519 
 9520   ins_pipe(pipe_slow);
 9521 %}
 9522 
 9523 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9524 
 9525   predicate(needs_acquiring_load_exclusive(n));
 9526   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9527   ins_cost(VOLATILE_REF_COST);
 9528 
 9529   effect(KILL cr);
 9530 
 9531  format %{
 9532     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9533     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9534  %}
 9535 
 9536  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9537             aarch64_enc_cset_eq(res));
 9538 
 9539   ins_pipe(pipe_slow);
 9540 %}
 9541 
 9542 
 9543 // ---------------------------------------------------------------------
 9544 
 9545 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9546 
 9547 // Sundry CAS operations.  Note that release is always true,
 9548 // regardless of the memory ordering of the CAS.  This is because we
 9549 // need the volatile case to be sequentially consistent but there is
 9550 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9551 // can't check the type of memory ordering here, so we always emit a
 9552 // STLXR.
 9553 
 9554 // This section is generated from aarch64_ad_cas.m4
 9555 
 9556 
 9557 
 9558 // This pattern is generated automatically from cas.m4.
 9559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9560 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9561 
 9562   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9563   ins_cost(2 * VOLATILE_REF_COST);
 9564   effect(TEMP_DEF res, KILL cr);
 9565   format %{
 9566     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9567   %}
 9568   ins_encode %{
 9569     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9570                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9571                /*weak*/ false, $res$$Register);
 9572     __ sxtbw($res$$Register, $res$$Register);
 9573   %}
 9574   ins_pipe(pipe_slow);
 9575 %}
 9576 
 9577 // This pattern is generated automatically from cas.m4.
 9578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9579 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9580 
 9581   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9582   ins_cost(2 * VOLATILE_REF_COST);
 9583   effect(TEMP_DEF res, KILL cr);
 9584   format %{
 9585     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9586   %}
 9587   ins_encode %{
 9588     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9589                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9590                /*weak*/ false, $res$$Register);
 9591     __ sxthw($res$$Register, $res$$Register);
 9592   %}
 9593   ins_pipe(pipe_slow);
 9594 %}
 9595 
 9596 // This pattern is generated automatically from cas.m4.
 9597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9598 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9599 
 9600   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9601   ins_cost(2 * VOLATILE_REF_COST);
 9602   effect(TEMP_DEF res, KILL cr);
 9603   format %{
 9604     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9605   %}
 9606   ins_encode %{
 9607     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9608                Assembler::word, /*acquire*/ false, /*release*/ true,
 9609                /*weak*/ false, $res$$Register);
 9610   %}
 9611   ins_pipe(pipe_slow);
 9612 %}
 9613 
 9614 // This pattern is generated automatically from cas.m4.
 9615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9616 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9617 
 9618   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9619   ins_cost(2 * VOLATILE_REF_COST);
 9620   effect(TEMP_DEF res, KILL cr);
 9621   format %{
 9622     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9623   %}
 9624   ins_encode %{
 9625     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9626                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9627                /*weak*/ false, $res$$Register);
 9628   %}
 9629   ins_pipe(pipe_slow);
 9630 %}
 9631 
 9632 // This pattern is generated automatically from cas.m4.
 9633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9634 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9635 
 9636   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9637   ins_cost(2 * VOLATILE_REF_COST);
 9638   effect(TEMP_DEF res, KILL cr);
 9639   format %{
 9640     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9641   %}
 9642   ins_encode %{
 9643     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9644                Assembler::word, /*acquire*/ false, /*release*/ true,
 9645                /*weak*/ false, $res$$Register);
 9646   %}
 9647   ins_pipe(pipe_slow);
 9648 %}
 9649 
 9650 // This pattern is generated automatically from cas.m4.
 9651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9652 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9653   predicate(n->as_LoadStore()->barrier_data() == 0);
 9654   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9655   ins_cost(2 * VOLATILE_REF_COST);
 9656   effect(TEMP_DEF res, KILL cr);
 9657   format %{
 9658     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9659   %}
 9660   ins_encode %{
 9661     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9662                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9663                /*weak*/ false, $res$$Register);
 9664   %}
 9665   ins_pipe(pipe_slow);
 9666 %}
 9667 
 9668 // This pattern is generated automatically from cas.m4.
 9669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9670 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9671   predicate(needs_acquiring_load_exclusive(n));
 9672   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9673   ins_cost(VOLATILE_REF_COST);
 9674   effect(TEMP_DEF res, KILL cr);
 9675   format %{
 9676     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9677   %}
 9678   ins_encode %{
 9679     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9680                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9681                /*weak*/ false, $res$$Register);
 9682     __ sxtbw($res$$Register, $res$$Register);
 9683   %}
 9684   ins_pipe(pipe_slow);
 9685 %}
 9686 
 9687 // This pattern is generated automatically from cas.m4.
 9688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9689 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9690   predicate(needs_acquiring_load_exclusive(n));
 9691   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9692   ins_cost(VOLATILE_REF_COST);
 9693   effect(TEMP_DEF res, KILL cr);
 9694   format %{
 9695     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9696   %}
 9697   ins_encode %{
 9698     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9699                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9700                /*weak*/ false, $res$$Register);
 9701     __ sxthw($res$$Register, $res$$Register);
 9702   %}
 9703   ins_pipe(pipe_slow);
 9704 %}
 9705 
 9706 // This pattern is generated automatically from cas.m4.
 9707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9708 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9709   predicate(needs_acquiring_load_exclusive(n));
 9710   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9711   ins_cost(VOLATILE_REF_COST);
 9712   effect(TEMP_DEF res, KILL cr);
 9713   format %{
 9714     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9715   %}
 9716   ins_encode %{
 9717     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9718                Assembler::word, /*acquire*/ true, /*release*/ true,
 9719                /*weak*/ false, $res$$Register);
 9720   %}
 9721   ins_pipe(pipe_slow);
 9722 %}
 9723 
 9724 // This pattern is generated automatically from cas.m4.
 9725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9726 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9727   predicate(needs_acquiring_load_exclusive(n));
 9728   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9729   ins_cost(VOLATILE_REF_COST);
 9730   effect(TEMP_DEF res, KILL cr);
 9731   format %{
 9732     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9733   %}
 9734   ins_encode %{
 9735     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9736                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9737                /*weak*/ false, $res$$Register);
 9738   %}
 9739   ins_pipe(pipe_slow);
 9740 %}
 9741 
 9742 // This pattern is generated automatically from cas.m4.
 9743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9744 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9745   predicate(needs_acquiring_load_exclusive(n));
 9746   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9747   ins_cost(VOLATILE_REF_COST);
 9748   effect(TEMP_DEF res, KILL cr);
 9749   format %{
 9750     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9751   %}
 9752   ins_encode %{
 9753     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9754                Assembler::word, /*acquire*/ true, /*release*/ true,
 9755                /*weak*/ false, $res$$Register);
 9756   %}
 9757   ins_pipe(pipe_slow);
 9758 %}
 9759 
 9760 // This pattern is generated automatically from cas.m4.
 9761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9762 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9763   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9764   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9765   ins_cost(VOLATILE_REF_COST);
 9766   effect(TEMP_DEF res, KILL cr);
 9767   format %{
 9768     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9769   %}
 9770   ins_encode %{
 9771     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9772                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9773                /*weak*/ false, $res$$Register);
 9774   %}
 9775   ins_pipe(pipe_slow);
 9776 %}
 9777 
 9778 // This pattern is generated automatically from cas.m4.
 9779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9780 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9781 
 9782   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9783   ins_cost(2 * VOLATILE_REF_COST);
 9784   effect(KILL cr);
 9785   format %{
 9786     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9787     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9788   %}
 9789   ins_encode %{
 9790     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9791                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9792                /*weak*/ true, noreg);
 9793     __ csetw($res$$Register, Assembler::EQ);
 9794   %}
 9795   ins_pipe(pipe_slow);
 9796 %}
 9797 
 9798 // This pattern is generated automatically from cas.m4.
 9799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9800 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9801 
 9802   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9803   ins_cost(2 * VOLATILE_REF_COST);
 9804   effect(KILL cr);
 9805   format %{
 9806     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9807     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9808   %}
 9809   ins_encode %{
 9810     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9811                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9812                /*weak*/ true, noreg);
 9813     __ csetw($res$$Register, Assembler::EQ);
 9814   %}
 9815   ins_pipe(pipe_slow);
 9816 %}
 9817 
 9818 // This pattern is generated automatically from cas.m4.
 9819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9820 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9821 
 9822   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9823   ins_cost(2 * VOLATILE_REF_COST);
 9824   effect(KILL cr);
 9825   format %{
 9826     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9827     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9828   %}
 9829   ins_encode %{
 9830     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9831                Assembler::word, /*acquire*/ false, /*release*/ true,
 9832                /*weak*/ true, noreg);
 9833     __ csetw($res$$Register, Assembler::EQ);
 9834   %}
 9835   ins_pipe(pipe_slow);
 9836 %}
 9837 
 9838 // This pattern is generated automatically from cas.m4.
 9839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9840 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9841 
 9842   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9843   ins_cost(2 * VOLATILE_REF_COST);
 9844   effect(KILL cr);
 9845   format %{
 9846     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9847     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9848   %}
 9849   ins_encode %{
 9850     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9851                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9852                /*weak*/ true, noreg);
 9853     __ csetw($res$$Register, Assembler::EQ);
 9854   %}
 9855   ins_pipe(pipe_slow);
 9856 %}
 9857 
 9858 // This pattern is generated automatically from cas.m4.
 9859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9860 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9861 
 9862   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9863   ins_cost(2 * VOLATILE_REF_COST);
 9864   effect(KILL cr);
 9865   format %{
 9866     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9867     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9868   %}
 9869   ins_encode %{
 9870     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9871                Assembler::word, /*acquire*/ false, /*release*/ true,
 9872                /*weak*/ true, noreg);
 9873     __ csetw($res$$Register, Assembler::EQ);
 9874   %}
 9875   ins_pipe(pipe_slow);
 9876 %}
 9877 
 9878 // This pattern is generated automatically from cas.m4.
 9879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9880 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9881   predicate(n->as_LoadStore()->barrier_data() == 0);
 9882   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9883   ins_cost(2 * VOLATILE_REF_COST);
 9884   effect(KILL cr);
 9885   format %{
 9886     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9887     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9888   %}
 9889   ins_encode %{
 9890     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9891                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9892                /*weak*/ true, noreg);
 9893     __ csetw($res$$Register, Assembler::EQ);
 9894   %}
 9895   ins_pipe(pipe_slow);
 9896 %}
 9897 
 9898 // This pattern is generated automatically from cas.m4.
 9899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9900 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9901   predicate(needs_acquiring_load_exclusive(n));
 9902   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9903   ins_cost(VOLATILE_REF_COST);
 9904   effect(KILL cr);
 9905   format %{
 9906     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9907     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9908   %}
 9909   ins_encode %{
 9910     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9911                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9912                /*weak*/ true, noreg);
 9913     __ csetw($res$$Register, Assembler::EQ);
 9914   %}
 9915   ins_pipe(pipe_slow);
 9916 %}
 9917 
 9918 // This pattern is generated automatically from cas.m4.
 9919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9920 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9921   predicate(needs_acquiring_load_exclusive(n));
 9922   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9923   ins_cost(VOLATILE_REF_COST);
 9924   effect(KILL cr);
 9925   format %{
 9926     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9927     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9928   %}
 9929   ins_encode %{
 9930     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9931                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9932                /*weak*/ true, noreg);
 9933     __ csetw($res$$Register, Assembler::EQ);
 9934   %}
 9935   ins_pipe(pipe_slow);
 9936 %}
 9937 
 9938 // This pattern is generated automatically from cas.m4.
 9939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9940 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9941   predicate(needs_acquiring_load_exclusive(n));
 9942   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9943   ins_cost(VOLATILE_REF_COST);
 9944   effect(KILL cr);
 9945   format %{
 9946     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9947     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9948   %}
 9949   ins_encode %{
 9950     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9951                Assembler::word, /*acquire*/ true, /*release*/ true,
 9952                /*weak*/ true, noreg);
 9953     __ csetw($res$$Register, Assembler::EQ);
 9954   %}
 9955   ins_pipe(pipe_slow);
 9956 %}
 9957 
 9958 // This pattern is generated automatically from cas.m4.
 9959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9960 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9961   predicate(needs_acquiring_load_exclusive(n));
 9962   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9963   ins_cost(VOLATILE_REF_COST);
 9964   effect(KILL cr);
 9965   format %{
 9966     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9967     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9968   %}
 9969   ins_encode %{
 9970     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9971                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9972                /*weak*/ true, noreg);
 9973     __ csetw($res$$Register, Assembler::EQ);
 9974   %}
 9975   ins_pipe(pipe_slow);
 9976 %}
 9977 
 9978 // This pattern is generated automatically from cas.m4.
 9979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9980 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9981   predicate(needs_acquiring_load_exclusive(n));
 9982   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9983   ins_cost(VOLATILE_REF_COST);
 9984   effect(KILL cr);
 9985   format %{
 9986     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9987     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9988   %}
 9989   ins_encode %{
 9990     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9991                Assembler::word, /*acquire*/ true, /*release*/ true,
 9992                /*weak*/ true, noreg);
 9993     __ csetw($res$$Register, Assembler::EQ);
 9994   %}
 9995   ins_pipe(pipe_slow);
 9996 %}
 9997 
 9998 // This pattern is generated automatically from cas.m4.
 9999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10000 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
10001   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
10002   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10003   ins_cost(VOLATILE_REF_COST);
10004   effect(KILL cr);
10005   format %{
10006     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10007     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10008   %}
10009   ins_encode %{
10010     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10011                Assembler::xword, /*acquire*/ true, /*release*/ true,
10012                /*weak*/ true, noreg);
10013     __ csetw($res$$Register, Assembler::EQ);
10014   %}
10015   ins_pipe(pipe_slow);
10016 %}
10017 
10018 // END This section of the file is automatically generated. Do not edit --------------
10019 // ---------------------------------------------------------------------
10020 
10021 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
10022   match(Set prev (GetAndSetI mem newv));
10023   ins_cost(2 * VOLATILE_REF_COST);
10024   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
10025   ins_encode %{
10026     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10027   %}
10028   ins_pipe(pipe_serial);
10029 %}
10030 
10031 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
10032   match(Set prev (GetAndSetL mem newv));
10033   ins_cost(2 * VOLATILE_REF_COST);
10034   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
10035   ins_encode %{
10036     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
10037   %}
10038   ins_pipe(pipe_serial);
10039 %}
10040 
10041 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
10042   match(Set prev (GetAndSetN mem newv));
10043   ins_cost(2 * VOLATILE_REF_COST);
10044   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
10045   ins_encode %{
10046     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10047   %}
10048   ins_pipe(pipe_serial);
10049 %}
10050 
10051 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
10052   predicate(n->as_LoadStore()->barrier_data() == 0);
10053   match(Set prev (GetAndSetP mem newv));
10054   ins_cost(2 * VOLATILE_REF_COST);
10055   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
10056   ins_encode %{
10057     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
10058   %}
10059   ins_pipe(pipe_serial);
10060 %}
10061 
10062 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
10063   predicate(needs_acquiring_load_exclusive(n));
10064   match(Set prev (GetAndSetI mem newv));
10065   ins_cost(VOLATILE_REF_COST);
10066   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
10067   ins_encode %{
10068     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10069   %}
10070   ins_pipe(pipe_serial);
10071 %}
10072 
10073 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
10074   predicate(needs_acquiring_load_exclusive(n));
10075   match(Set prev (GetAndSetL mem newv));
10076   ins_cost(VOLATILE_REF_COST);
10077   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
10078   ins_encode %{
10079     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10080   %}
10081   ins_pipe(pipe_serial);
10082 %}
10083 
10084 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
10085   predicate(needs_acquiring_load_exclusive(n));
10086   match(Set prev (GetAndSetN mem newv));
10087   ins_cost(VOLATILE_REF_COST);
10088   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
10089   ins_encode %{
10090     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10091   %}
10092   ins_pipe(pipe_serial);
10093 %}
10094 
10095 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
10096   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
10097   match(Set prev (GetAndSetP mem newv));
10098   ins_cost(VOLATILE_REF_COST);
10099   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
10100   ins_encode %{
10101     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10102   %}
10103   ins_pipe(pipe_serial);
10104 %}
10105 
10106 
10107 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
10108   match(Set newval (GetAndAddL mem incr));
10109   ins_cost(2 * VOLATILE_REF_COST + 1);
10110   format %{ "get_and_addL $newval, [$mem], $incr" %}
10111   ins_encode %{
10112     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
10113   %}
10114   ins_pipe(pipe_serial);
10115 %}
10116 
10117 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
10118   predicate(n->as_LoadStore()->result_not_used());
10119   match(Set dummy (GetAndAddL mem incr));
10120   ins_cost(2 * VOLATILE_REF_COST);
10121   format %{ "get_and_addL [$mem], $incr" %}
10122   ins_encode %{
10123     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
10124   %}
10125   ins_pipe(pipe_serial);
10126 %}
10127 
10128 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10129   match(Set newval (GetAndAddL mem incr));
10130   ins_cost(2 * VOLATILE_REF_COST + 1);
10131   format %{ "get_and_addL $newval, [$mem], $incr" %}
10132   ins_encode %{
10133     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
10134   %}
10135   ins_pipe(pipe_serial);
10136 %}
10137 
10138 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
10139   predicate(n->as_LoadStore()->result_not_used());
10140   match(Set dummy (GetAndAddL mem incr));
10141   ins_cost(2 * VOLATILE_REF_COST);
10142   format %{ "get_and_addL [$mem], $incr" %}
10143   ins_encode %{
10144     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
10145   %}
10146   ins_pipe(pipe_serial);
10147 %}
10148 
10149 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10150   match(Set newval (GetAndAddI mem incr));
10151   ins_cost(2 * VOLATILE_REF_COST + 1);
10152   format %{ "get_and_addI $newval, [$mem], $incr" %}
10153   ins_encode %{
10154     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10155   %}
10156   ins_pipe(pipe_serial);
10157 %}
10158 
10159 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10160   predicate(n->as_LoadStore()->result_not_used());
10161   match(Set dummy (GetAndAddI mem incr));
10162   ins_cost(2 * VOLATILE_REF_COST);
10163   format %{ "get_and_addI [$mem], $incr" %}
10164   ins_encode %{
10165     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10166   %}
10167   ins_pipe(pipe_serial);
10168 %}
10169 
10170 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10171   match(Set newval (GetAndAddI mem incr));
10172   ins_cost(2 * VOLATILE_REF_COST + 1);
10173   format %{ "get_and_addI $newval, [$mem], $incr" %}
10174   ins_encode %{
10175     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10176   %}
10177   ins_pipe(pipe_serial);
10178 %}
10179 
10180 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10181   predicate(n->as_LoadStore()->result_not_used());
10182   match(Set dummy (GetAndAddI mem incr));
10183   ins_cost(2 * VOLATILE_REF_COST);
10184   format %{ "get_and_addI [$mem], $incr" %}
10185   ins_encode %{
10186     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10187   %}
10188   ins_pipe(pipe_serial);
10189 %}
10190 
10191 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10192   predicate(needs_acquiring_load_exclusive(n));
10193   match(Set newval (GetAndAddL mem incr));
10194   ins_cost(VOLATILE_REF_COST + 1);
10195   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10196   ins_encode %{
10197     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10198   %}
10199   ins_pipe(pipe_serial);
10200 %}
10201 
10202 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10203   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10204   match(Set dummy (GetAndAddL mem incr));
10205   ins_cost(VOLATILE_REF_COST);
10206   format %{ "get_and_addL_acq [$mem], $incr" %}
10207   ins_encode %{
10208     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10209   %}
10210   ins_pipe(pipe_serial);
10211 %}
10212 
10213 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10214   predicate(needs_acquiring_load_exclusive(n));
10215   match(Set newval (GetAndAddL mem incr));
10216   ins_cost(VOLATILE_REF_COST + 1);
10217   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10218   ins_encode %{
10219     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10220   %}
10221   ins_pipe(pipe_serial);
10222 %}
10223 
10224 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10225   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10226   match(Set dummy (GetAndAddL mem incr));
10227   ins_cost(VOLATILE_REF_COST);
10228   format %{ "get_and_addL_acq [$mem], $incr" %}
10229   ins_encode %{
10230     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10231   %}
10232   ins_pipe(pipe_serial);
10233 %}
10234 
10235 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10236   predicate(needs_acquiring_load_exclusive(n));
10237   match(Set newval (GetAndAddI mem incr));
10238   ins_cost(VOLATILE_REF_COST + 1);
10239   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10240   ins_encode %{
10241     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10242   %}
10243   ins_pipe(pipe_serial);
10244 %}
10245 
10246 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10247   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10248   match(Set dummy (GetAndAddI mem incr));
10249   ins_cost(VOLATILE_REF_COST);
10250   format %{ "get_and_addI_acq [$mem], $incr" %}
10251   ins_encode %{
10252     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10253   %}
10254   ins_pipe(pipe_serial);
10255 %}
10256 
10257 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10258   predicate(needs_acquiring_load_exclusive(n));
10259   match(Set newval (GetAndAddI mem incr));
10260   ins_cost(VOLATILE_REF_COST + 1);
10261   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10262   ins_encode %{
10263     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10264   %}
10265   ins_pipe(pipe_serial);
10266 %}
10267 
10268 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10269   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10270   match(Set dummy (GetAndAddI mem incr));
10271   ins_cost(VOLATILE_REF_COST);
10272   format %{ "get_and_addI_acq [$mem], $incr" %}
10273   ins_encode %{
10274     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10275   %}
10276   ins_pipe(pipe_serial);
10277 %}
10278 
10279 // Manifest a CmpL result in an integer register.
10280 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10281 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10282 %{
10283   match(Set dst (CmpL3 src1 src2));
10284   effect(KILL flags);
10285 
10286   ins_cost(INSN_COST * 6);
10287   format %{
10288       "cmp $src1, $src2"
10289       "csetw $dst, ne"
10290       "cnegw $dst, lt"
10291   %}
10292   // format %{ "CmpL3 $dst, $src1, $src2" %}
10293   ins_encode %{
10294     __ cmp($src1$$Register, $src2$$Register);
10295     __ csetw($dst$$Register, Assembler::NE);
10296     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10297   %}
10298 
10299   ins_pipe(pipe_class_default);
10300 %}
10301 
10302 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10303 %{
10304   match(Set dst (CmpL3 src1 src2));
10305   effect(KILL flags);
10306 
10307   ins_cost(INSN_COST * 6);
10308   format %{
10309       "cmp $src1, $src2"
10310       "csetw $dst, ne"
10311       "cnegw $dst, lt"
10312   %}
10313   ins_encode %{
10314     int32_t con = (int32_t)$src2$$constant;
10315      if (con < 0) {
10316       __ adds(zr, $src1$$Register, -con);
10317     } else {
10318       __ subs(zr, $src1$$Register, con);
10319     }
10320     __ csetw($dst$$Register, Assembler::NE);
10321     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10322   %}
10323 
10324   ins_pipe(pipe_class_default);
10325 %}
10326 
10327 // ============================================================================
10328 // Conditional Move Instructions
10329 
10330 // n.b. we have identical rules for both a signed compare op (cmpOp)
10331 // and an unsigned compare op (cmpOpU). it would be nice if we could
10332 // define an op class which merged both inputs and use it to type the
10333 // argument to a single rule. unfortunatelyt his fails because the
10334 // opclass does not live up to the COND_INTER interface of its
10335 // component operands. When the generic code tries to negate the
10336 // operand it ends up running the generci Machoper::negate method
10337 // which throws a ShouldNotHappen. So, we have to provide two flavours
10338 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10339 
10340 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10341   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10342 
10343   ins_cost(INSN_COST * 2);
10344   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10345 
10346   ins_encode %{
10347     __ cselw(as_Register($dst$$reg),
10348              as_Register($src2$$reg),
10349              as_Register($src1$$reg),
10350              (Assembler::Condition)$cmp$$cmpcode);
10351   %}
10352 
10353   ins_pipe(icond_reg_reg);
10354 %}
10355 
10356 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10357   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10358 
10359   ins_cost(INSN_COST * 2);
10360   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10361 
10362   ins_encode %{
10363     __ cselw(as_Register($dst$$reg),
10364              as_Register($src2$$reg),
10365              as_Register($src1$$reg),
10366              (Assembler::Condition)$cmp$$cmpcode);
10367   %}
10368 
10369   ins_pipe(icond_reg_reg);
10370 %}
10371 
10372 // special cases where one arg is zero
10373 
10374 // n.b. this is selected in preference to the rule above because it
10375 // avoids loading constant 0 into a source register
10376 
10377 // TODO
10378 // we ought only to be able to cull one of these variants as the ideal
10379 // transforms ought always to order the zero consistently (to left/right?)
10380 
10381 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10382   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10383 
10384   ins_cost(INSN_COST * 2);
10385   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10386 
10387   ins_encode %{
10388     __ cselw(as_Register($dst$$reg),
10389              as_Register($src$$reg),
10390              zr,
10391              (Assembler::Condition)$cmp$$cmpcode);
10392   %}
10393 
10394   ins_pipe(icond_reg);
10395 %}
10396 
10397 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10398   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10399 
10400   ins_cost(INSN_COST * 2);
10401   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10402 
10403   ins_encode %{
10404     __ cselw(as_Register($dst$$reg),
10405              as_Register($src$$reg),
10406              zr,
10407              (Assembler::Condition)$cmp$$cmpcode);
10408   %}
10409 
10410   ins_pipe(icond_reg);
10411 %}
10412 
10413 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10414   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10415 
10416   ins_cost(INSN_COST * 2);
10417   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10418 
10419   ins_encode %{
10420     __ cselw(as_Register($dst$$reg),
10421              zr,
10422              as_Register($src$$reg),
10423              (Assembler::Condition)$cmp$$cmpcode);
10424   %}
10425 
10426   ins_pipe(icond_reg);
10427 %}
10428 
10429 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10430   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10431 
10432   ins_cost(INSN_COST * 2);
10433   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10434 
10435   ins_encode %{
10436     __ cselw(as_Register($dst$$reg),
10437              zr,
10438              as_Register($src$$reg),
10439              (Assembler::Condition)$cmp$$cmpcode);
10440   %}
10441 
10442   ins_pipe(icond_reg);
10443 %}
10444 
10445 // special case for creating a boolean 0 or 1
10446 
10447 // n.b. this is selected in preference to the rule above because it
10448 // avoids loading constants 0 and 1 into a source register
10449 
10450 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10451   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10452 
10453   ins_cost(INSN_COST * 2);
10454   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10455 
10456   ins_encode %{
10457     // equivalently
10458     // cset(as_Register($dst$$reg),
10459     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10460     __ csincw(as_Register($dst$$reg),
10461              zr,
10462              zr,
10463              (Assembler::Condition)$cmp$$cmpcode);
10464   %}
10465 
10466   ins_pipe(icond_none);
10467 %}
10468 
10469 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10470   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10471 
10472   ins_cost(INSN_COST * 2);
10473   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10474 
10475   ins_encode %{
10476     // equivalently
10477     // cset(as_Register($dst$$reg),
10478     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10479     __ csincw(as_Register($dst$$reg),
10480              zr,
10481              zr,
10482              (Assembler::Condition)$cmp$$cmpcode);
10483   %}
10484 
10485   ins_pipe(icond_none);
10486 %}
10487 
10488 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10489   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10490 
10491   ins_cost(INSN_COST * 2);
10492   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10493 
10494   ins_encode %{
10495     __ csel(as_Register($dst$$reg),
10496             as_Register($src2$$reg),
10497             as_Register($src1$$reg),
10498             (Assembler::Condition)$cmp$$cmpcode);
10499   %}
10500 
10501   ins_pipe(icond_reg_reg);
10502 %}
10503 
10504 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10505   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10506 
10507   ins_cost(INSN_COST * 2);
10508   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10509 
10510   ins_encode %{
10511     __ csel(as_Register($dst$$reg),
10512             as_Register($src2$$reg),
10513             as_Register($src1$$reg),
10514             (Assembler::Condition)$cmp$$cmpcode);
10515   %}
10516 
10517   ins_pipe(icond_reg_reg);
10518 %}
10519 
10520 // special cases where one arg is zero
10521 
10522 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10523   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10524 
10525   ins_cost(INSN_COST * 2);
10526   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10527 
10528   ins_encode %{
10529     __ csel(as_Register($dst$$reg),
10530             zr,
10531             as_Register($src$$reg),
10532             (Assembler::Condition)$cmp$$cmpcode);
10533   %}
10534 
10535   ins_pipe(icond_reg);
10536 %}
10537 
10538 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10539   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10540 
10541   ins_cost(INSN_COST * 2);
10542   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10543 
10544   ins_encode %{
10545     __ csel(as_Register($dst$$reg),
10546             zr,
10547             as_Register($src$$reg),
10548             (Assembler::Condition)$cmp$$cmpcode);
10549   %}
10550 
10551   ins_pipe(icond_reg);
10552 %}
10553 
10554 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10555   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10556 
10557   ins_cost(INSN_COST * 2);
10558   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10559 
10560   ins_encode %{
10561     __ csel(as_Register($dst$$reg),
10562             as_Register($src$$reg),
10563             zr,
10564             (Assembler::Condition)$cmp$$cmpcode);
10565   %}
10566 
10567   ins_pipe(icond_reg);
10568 %}
10569 
10570 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10571   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10572 
10573   ins_cost(INSN_COST * 2);
10574   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10575 
10576   ins_encode %{
10577     __ csel(as_Register($dst$$reg),
10578             as_Register($src$$reg),
10579             zr,
10580             (Assembler::Condition)$cmp$$cmpcode);
10581   %}
10582 
10583   ins_pipe(icond_reg);
10584 %}
10585 
10586 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10587   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10588 
10589   ins_cost(INSN_COST * 2);
10590   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10591 
10592   ins_encode %{
10593     __ csel(as_Register($dst$$reg),
10594             as_Register($src2$$reg),
10595             as_Register($src1$$reg),
10596             (Assembler::Condition)$cmp$$cmpcode);
10597   %}
10598 
10599   ins_pipe(icond_reg_reg);
10600 %}
10601 
10602 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10603   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10604 
10605   ins_cost(INSN_COST * 2);
10606   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10607 
10608   ins_encode %{
10609     __ csel(as_Register($dst$$reg),
10610             as_Register($src2$$reg),
10611             as_Register($src1$$reg),
10612             (Assembler::Condition)$cmp$$cmpcode);
10613   %}
10614 
10615   ins_pipe(icond_reg_reg);
10616 %}
10617 
10618 // special cases where one arg is zero
10619 
10620 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10621   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10622 
10623   ins_cost(INSN_COST * 2);
10624   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10625 
10626   ins_encode %{
10627     __ csel(as_Register($dst$$reg),
10628             zr,
10629             as_Register($src$$reg),
10630             (Assembler::Condition)$cmp$$cmpcode);
10631   %}
10632 
10633   ins_pipe(icond_reg);
10634 %}
10635 
10636 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10637   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10638 
10639   ins_cost(INSN_COST * 2);
10640   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10641 
10642   ins_encode %{
10643     __ csel(as_Register($dst$$reg),
10644             zr,
10645             as_Register($src$$reg),
10646             (Assembler::Condition)$cmp$$cmpcode);
10647   %}
10648 
10649   ins_pipe(icond_reg);
10650 %}
10651 
10652 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10653   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10654 
10655   ins_cost(INSN_COST * 2);
10656   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10657 
10658   ins_encode %{
10659     __ csel(as_Register($dst$$reg),
10660             as_Register($src$$reg),
10661             zr,
10662             (Assembler::Condition)$cmp$$cmpcode);
10663   %}
10664 
10665   ins_pipe(icond_reg);
10666 %}
10667 
10668 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10669   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10670 
10671   ins_cost(INSN_COST * 2);
10672   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10673 
10674   ins_encode %{
10675     __ csel(as_Register($dst$$reg),
10676             as_Register($src$$reg),
10677             zr,
10678             (Assembler::Condition)$cmp$$cmpcode);
10679   %}
10680 
10681   ins_pipe(icond_reg);
10682 %}
10683 
10684 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10685   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10686 
10687   ins_cost(INSN_COST * 2);
10688   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10689 
10690   ins_encode %{
10691     __ cselw(as_Register($dst$$reg),
10692              as_Register($src2$$reg),
10693              as_Register($src1$$reg),
10694              (Assembler::Condition)$cmp$$cmpcode);
10695   %}
10696 
10697   ins_pipe(icond_reg_reg);
10698 %}
10699 
10700 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10701   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10702 
10703   ins_cost(INSN_COST * 2);
10704   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10705 
10706   ins_encode %{
10707     __ cselw(as_Register($dst$$reg),
10708              as_Register($src2$$reg),
10709              as_Register($src1$$reg),
10710              (Assembler::Condition)$cmp$$cmpcode);
10711   %}
10712 
10713   ins_pipe(icond_reg_reg);
10714 %}
10715 
10716 // special cases where one arg is zero
10717 
10718 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10719   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10720 
10721   ins_cost(INSN_COST * 2);
10722   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10723 
10724   ins_encode %{
10725     __ cselw(as_Register($dst$$reg),
10726              zr,
10727              as_Register($src$$reg),
10728              (Assembler::Condition)$cmp$$cmpcode);
10729   %}
10730 
10731   ins_pipe(icond_reg);
10732 %}
10733 
10734 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10735   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10736 
10737   ins_cost(INSN_COST * 2);
10738   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10739 
10740   ins_encode %{
10741     __ cselw(as_Register($dst$$reg),
10742              zr,
10743              as_Register($src$$reg),
10744              (Assembler::Condition)$cmp$$cmpcode);
10745   %}
10746 
10747   ins_pipe(icond_reg);
10748 %}
10749 
10750 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10751   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10752 
10753   ins_cost(INSN_COST * 2);
10754   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10755 
10756   ins_encode %{
10757     __ cselw(as_Register($dst$$reg),
10758              as_Register($src$$reg),
10759              zr,
10760              (Assembler::Condition)$cmp$$cmpcode);
10761   %}
10762 
10763   ins_pipe(icond_reg);
10764 %}
10765 
10766 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10767   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10768 
10769   ins_cost(INSN_COST * 2);
10770   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10771 
10772   ins_encode %{
10773     __ cselw(as_Register($dst$$reg),
10774              as_Register($src$$reg),
10775              zr,
10776              (Assembler::Condition)$cmp$$cmpcode);
10777   %}
10778 
10779   ins_pipe(icond_reg);
10780 %}
10781 
10782 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10783 %{
10784   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10785 
10786   ins_cost(INSN_COST * 3);
10787 
10788   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10789   ins_encode %{
10790     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10791     __ fcsels(as_FloatRegister($dst$$reg),
10792               as_FloatRegister($src2$$reg),
10793               as_FloatRegister($src1$$reg),
10794               cond);
10795   %}
10796 
10797   ins_pipe(fp_cond_reg_reg_s);
10798 %}
10799 
10800 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10801 %{
10802   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10803 
10804   ins_cost(INSN_COST * 3);
10805 
10806   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10807   ins_encode %{
10808     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10809     __ fcsels(as_FloatRegister($dst$$reg),
10810               as_FloatRegister($src2$$reg),
10811               as_FloatRegister($src1$$reg),
10812               cond);
10813   %}
10814 
10815   ins_pipe(fp_cond_reg_reg_s);
10816 %}
10817 
10818 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10819 %{
10820   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10821 
10822   ins_cost(INSN_COST * 3);
10823 
10824   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10825   ins_encode %{
10826     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10827     __ fcseld(as_FloatRegister($dst$$reg),
10828               as_FloatRegister($src2$$reg),
10829               as_FloatRegister($src1$$reg),
10830               cond);
10831   %}
10832 
10833   ins_pipe(fp_cond_reg_reg_d);
10834 %}
10835 
10836 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10837 %{
10838   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10839 
10840   ins_cost(INSN_COST * 3);
10841 
10842   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10843   ins_encode %{
10844     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10845     __ fcseld(as_FloatRegister($dst$$reg),
10846               as_FloatRegister($src2$$reg),
10847               as_FloatRegister($src1$$reg),
10848               cond);
10849   %}
10850 
10851   ins_pipe(fp_cond_reg_reg_d);
10852 %}
10853 
10854 // ============================================================================
10855 // Arithmetic Instructions
10856 //
10857 
10858 // Integer Addition
10859 
10860 // TODO
10861 // these currently employ operations which do not set CR and hence are
10862 // not flagged as killing CR but we would like to isolate the cases
10863 // where we want to set flags from those where we don't. need to work
10864 // out how to do that.
10865 
10866 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10867   match(Set dst (AddI src1 src2));
10868 
10869   ins_cost(INSN_COST);
10870   format %{ "addw  $dst, $src1, $src2" %}
10871 
10872   ins_encode %{
10873     __ addw(as_Register($dst$$reg),
10874             as_Register($src1$$reg),
10875             as_Register($src2$$reg));
10876   %}
10877 
10878   ins_pipe(ialu_reg_reg);
10879 %}
10880 
10881 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10882   match(Set dst (AddI src1 src2));
10883 
10884   ins_cost(INSN_COST);
10885   format %{ "addw $dst, $src1, $src2" %}
10886 
10887   // use opcode to indicate that this is an add not a sub
10888   opcode(0x0);
10889 
10890   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10891 
10892   ins_pipe(ialu_reg_imm);
10893 %}
10894 
10895 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10896   match(Set dst (AddI (ConvL2I src1) src2));
10897 
10898   ins_cost(INSN_COST);
10899   format %{ "addw $dst, $src1, $src2" %}
10900 
10901   // use opcode to indicate that this is an add not a sub
10902   opcode(0x0);
10903 
10904   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10905 
10906   ins_pipe(ialu_reg_imm);
10907 %}
10908 
10909 // Pointer Addition
10910 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10911   match(Set dst (AddP src1 src2));
10912 
10913   ins_cost(INSN_COST);
10914   format %{ "add $dst, $src1, $src2\t# ptr" %}
10915 
10916   ins_encode %{
10917     __ add(as_Register($dst$$reg),
10918            as_Register($src1$$reg),
10919            as_Register($src2$$reg));
10920   %}
10921 
10922   ins_pipe(ialu_reg_reg);
10923 %}
10924 
10925 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10926   match(Set dst (AddP src1 (ConvI2L src2)));
10927 
10928   ins_cost(1.9 * INSN_COST);
10929   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10930 
10931   ins_encode %{
10932     __ add(as_Register($dst$$reg),
10933            as_Register($src1$$reg),
10934            as_Register($src2$$reg), ext::sxtw);
10935   %}
10936 
10937   ins_pipe(ialu_reg_reg);
10938 %}
10939 
10940 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10941   match(Set dst (AddP src1 (LShiftL src2 scale)));
10942 
10943   ins_cost(1.9 * INSN_COST);
10944   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10945 
10946   ins_encode %{
10947     __ lea(as_Register($dst$$reg),
10948            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10949                    Address::lsl($scale$$constant)));
10950   %}
10951 
10952   ins_pipe(ialu_reg_reg_shift);
10953 %}
10954 
10955 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10956   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10957 
10958   ins_cost(1.9 * INSN_COST);
10959   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10960 
10961   ins_encode %{
10962     __ lea(as_Register($dst$$reg),
10963            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10964                    Address::sxtw($scale$$constant)));
10965   %}
10966 
10967   ins_pipe(ialu_reg_reg_shift);
10968 %}
10969 
10970 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10971   match(Set dst (LShiftL (ConvI2L src) scale));
10972 
10973   ins_cost(INSN_COST);
10974   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10975 
10976   ins_encode %{
10977     __ sbfiz(as_Register($dst$$reg),
10978           as_Register($src$$reg),
10979           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10980   %}
10981 
10982   ins_pipe(ialu_reg_shift);
10983 %}
10984 
10985 // Pointer Immediate Addition
10986 // n.b. this needs to be more expensive than using an indirect memory
10987 // operand
10988 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10989   match(Set dst (AddP src1 src2));
10990 
10991   ins_cost(INSN_COST);
10992   format %{ "add $dst, $src1, $src2\t# ptr" %}
10993 
10994   // use opcode to indicate that this is an add not a sub
10995   opcode(0x0);
10996 
10997   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10998 
10999   ins_pipe(ialu_reg_imm);
11000 %}
11001 
11002 // Long Addition
11003 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11004 
11005   match(Set dst (AddL src1 src2));
11006 
11007   ins_cost(INSN_COST);
11008   format %{ "add  $dst, $src1, $src2" %}
11009 
11010   ins_encode %{
11011     __ add(as_Register($dst$$reg),
11012            as_Register($src1$$reg),
11013            as_Register($src2$$reg));
11014   %}
11015 
11016   ins_pipe(ialu_reg_reg);
11017 %}
11018 
11019 // No constant pool entries requiredLong Immediate Addition.
11020 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
11021   match(Set dst (AddL src1 src2));
11022 
11023   ins_cost(INSN_COST);
11024   format %{ "add $dst, $src1, $src2" %}
11025 
11026   // use opcode to indicate that this is an add not a sub
11027   opcode(0x0);
11028 
11029   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
11030 
11031   ins_pipe(ialu_reg_imm);
11032 %}
11033 
11034 // Integer Subtraction
11035 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11036   match(Set dst (SubI src1 src2));
11037 
11038   ins_cost(INSN_COST);
11039   format %{ "subw  $dst, $src1, $src2" %}
11040 
11041   ins_encode %{
11042     __ subw(as_Register($dst$$reg),
11043             as_Register($src1$$reg),
11044             as_Register($src2$$reg));
11045   %}
11046 
11047   ins_pipe(ialu_reg_reg);
11048 %}
11049 
11050 // Immediate Subtraction
11051 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
11052   match(Set dst (SubI src1 src2));
11053 
11054   ins_cost(INSN_COST);
11055   format %{ "subw $dst, $src1, $src2" %}
11056 
11057   // use opcode to indicate that this is a sub not an add
11058   opcode(0x1);
11059 
11060   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
11061 
11062   ins_pipe(ialu_reg_imm);
11063 %}
11064 
11065 // Long Subtraction
11066 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11067 
11068   match(Set dst (SubL src1 src2));
11069 
11070   ins_cost(INSN_COST);
11071   format %{ "sub  $dst, $src1, $src2" %}
11072 
11073   ins_encode %{
11074     __ sub(as_Register($dst$$reg),
11075            as_Register($src1$$reg),
11076            as_Register($src2$$reg));
11077   %}
11078 
11079   ins_pipe(ialu_reg_reg);
11080 %}
11081 
11082 // No constant pool entries requiredLong Immediate Subtraction.
11083 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
11084   match(Set dst (SubL src1 src2));
11085 
11086   ins_cost(INSN_COST);
11087   format %{ "sub$dst, $src1, $src2" %}
11088 
11089   // use opcode to indicate that this is a sub not an add
11090   opcode(0x1);
11091 
11092   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
11093 
11094   ins_pipe(ialu_reg_imm);
11095 %}
11096 
11097 // Integer Negation (special case for sub)
11098 
11099 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
11100   match(Set dst (SubI zero src));
11101 
11102   ins_cost(INSN_COST);
11103   format %{ "negw $dst, $src\t# int" %}
11104 
11105   ins_encode %{
11106     __ negw(as_Register($dst$$reg),
11107             as_Register($src$$reg));
11108   %}
11109 
11110   ins_pipe(ialu_reg);
11111 %}
11112 
11113 // Long Negation
11114 
11115 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
11116   match(Set dst (SubL zero src));
11117 
11118   ins_cost(INSN_COST);
11119   format %{ "neg $dst, $src\t# long" %}
11120 
11121   ins_encode %{
11122     __ neg(as_Register($dst$$reg),
11123            as_Register($src$$reg));
11124   %}
11125 
11126   ins_pipe(ialu_reg);
11127 %}
11128 
11129 // Integer Multiply
11130 
11131 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11132   match(Set dst (MulI src1 src2));
11133 
11134   ins_cost(INSN_COST * 3);
11135   format %{ "mulw  $dst, $src1, $src2" %}
11136 
11137   ins_encode %{
11138     __ mulw(as_Register($dst$$reg),
11139             as_Register($src1$$reg),
11140             as_Register($src2$$reg));
11141   %}
11142 
11143   ins_pipe(imul_reg_reg);
11144 %}
11145 
11146 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11147   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
11148 
11149   ins_cost(INSN_COST * 3);
11150   format %{ "smull  $dst, $src1, $src2" %}
11151 
11152   ins_encode %{
11153     __ smull(as_Register($dst$$reg),
11154              as_Register($src1$$reg),
11155              as_Register($src2$$reg));
11156   %}
11157 
11158   ins_pipe(imul_reg_reg);
11159 %}
11160 
11161 // Long Multiply
11162 
11163 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11164   match(Set dst (MulL src1 src2));
11165 
11166   ins_cost(INSN_COST * 5);
11167   format %{ "mul  $dst, $src1, $src2" %}
11168 
11169   ins_encode %{
11170     __ mul(as_Register($dst$$reg),
11171            as_Register($src1$$reg),
11172            as_Register($src2$$reg));
11173   %}
11174 
11175   ins_pipe(lmul_reg_reg);
11176 %}
11177 
11178 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11179 %{
11180   match(Set dst (MulHiL src1 src2));
11181 
11182   ins_cost(INSN_COST * 7);
11183   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
11184 
11185   ins_encode %{
11186     __ smulh(as_Register($dst$$reg),
11187              as_Register($src1$$reg),
11188              as_Register($src2$$reg));
11189   %}
11190 
11191   ins_pipe(lmul_reg_reg);
11192 %}
11193 
11194 // Combined Integer Multiply & Add/Sub
11195 
11196 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11197   match(Set dst (AddI src3 (MulI src1 src2)));
11198 
11199   ins_cost(INSN_COST * 3);
11200   format %{ "madd  $dst, $src1, $src2, $src3" %}
11201 
11202   ins_encode %{
11203     __ maddw(as_Register($dst$$reg),
11204              as_Register($src1$$reg),
11205              as_Register($src2$$reg),
11206              as_Register($src3$$reg));
11207   %}
11208 
11209   ins_pipe(imac_reg_reg);
11210 %}
11211 
11212 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11213   match(Set dst (SubI src3 (MulI src1 src2)));
11214 
11215   ins_cost(INSN_COST * 3);
11216   format %{ "msub  $dst, $src1, $src2, $src3" %}
11217 
11218   ins_encode %{
11219     __ msubw(as_Register($dst$$reg),
11220              as_Register($src1$$reg),
11221              as_Register($src2$$reg),
11222              as_Register($src3$$reg));
11223   %}
11224 
11225   ins_pipe(imac_reg_reg);
11226 %}
11227 
11228 // Combined Integer Multiply & Neg
11229 
11230 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11231   match(Set dst (MulI (SubI zero src1) src2));
11232 
11233   ins_cost(INSN_COST * 3);
11234   format %{ "mneg  $dst, $src1, $src2" %}
11235 
11236   ins_encode %{
11237     __ mnegw(as_Register($dst$$reg),
11238              as_Register($src1$$reg),
11239              as_Register($src2$$reg));
11240   %}
11241 
11242   ins_pipe(imac_reg_reg);
11243 %}
11244 
11245 // Combined Long Multiply & Add/Sub
11246 
11247 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11248   match(Set dst (AddL src3 (MulL src1 src2)));
11249 
11250   ins_cost(INSN_COST * 5);
11251   format %{ "madd  $dst, $src1, $src2, $src3" %}
11252 
11253   ins_encode %{
11254     __ madd(as_Register($dst$$reg),
11255             as_Register($src1$$reg),
11256             as_Register($src2$$reg),
11257             as_Register($src3$$reg));
11258   %}
11259 
11260   ins_pipe(lmac_reg_reg);
11261 %}
11262 
11263 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11264   match(Set dst (SubL src3 (MulL src1 src2)));
11265 
11266   ins_cost(INSN_COST * 5);
11267   format %{ "msub  $dst, $src1, $src2, $src3" %}
11268 
11269   ins_encode %{
11270     __ msub(as_Register($dst$$reg),
11271             as_Register($src1$$reg),
11272             as_Register($src2$$reg),
11273             as_Register($src3$$reg));
11274   %}
11275 
11276   ins_pipe(lmac_reg_reg);
11277 %}
11278 
11279 // Combined Long Multiply & Neg
11280 
11281 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11282   match(Set dst (MulL (SubL zero src1) src2));
11283 
11284   ins_cost(INSN_COST * 5);
11285   format %{ "mneg  $dst, $src1, $src2" %}
11286 
11287   ins_encode %{
11288     __ mneg(as_Register($dst$$reg),
11289             as_Register($src1$$reg),
11290             as_Register($src2$$reg));
11291   %}
11292 
11293   ins_pipe(lmac_reg_reg);
11294 %}
11295 
11296 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11297 
11298 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11299   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11300 
11301   ins_cost(INSN_COST * 3);
11302   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11303 
11304   ins_encode %{
11305     __ smaddl(as_Register($dst$$reg),
11306               as_Register($src1$$reg),
11307               as_Register($src2$$reg),
11308               as_Register($src3$$reg));
11309   %}
11310 
11311   ins_pipe(imac_reg_reg);
11312 %}
11313 
11314 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11315   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11316 
11317   ins_cost(INSN_COST * 3);
11318   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11319 
11320   ins_encode %{
11321     __ smsubl(as_Register($dst$$reg),
11322               as_Register($src1$$reg),
11323               as_Register($src2$$reg),
11324               as_Register($src3$$reg));
11325   %}
11326 
11327   ins_pipe(imac_reg_reg);
11328 %}
11329 
11330 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11331   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11332 
11333   ins_cost(INSN_COST * 3);
11334   format %{ "smnegl  $dst, $src1, $src2" %}
11335 
11336   ins_encode %{
11337     __ smnegl(as_Register($dst$$reg),
11338               as_Register($src1$$reg),
11339               as_Register($src2$$reg));
11340   %}
11341 
11342   ins_pipe(imac_reg_reg);
11343 %}
11344 
11345 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11346 
11347 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11348   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11349 
11350   ins_cost(INSN_COST * 5);
11351   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11352             "maddw $dst, $src3, $src4, rscratch1" %}
11353 
11354   ins_encode %{
11355     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11356     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11357 
11358   ins_pipe(imac_reg_reg);
11359 %}
11360 
11361 // Integer Divide
11362 
11363 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11364   match(Set dst (DivI src1 src2));
11365 
11366   ins_cost(INSN_COST * 19);
11367   format %{ "sdivw  $dst, $src1, $src2" %}
11368 
11369   ins_encode(aarch64_enc_divw(dst, src1, src2));
11370   ins_pipe(idiv_reg_reg);
11371 %}
11372 
11373 // Long Divide
11374 
11375 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11376   match(Set dst (DivL src1 src2));
11377 
11378   ins_cost(INSN_COST * 35);
11379   format %{ "sdiv   $dst, $src1, $src2" %}
11380 
11381   ins_encode(aarch64_enc_div(dst, src1, src2));
11382   ins_pipe(ldiv_reg_reg);
11383 %}
11384 
11385 // Integer Remainder
11386 
11387 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11388   match(Set dst (ModI src1 src2));
11389 
11390   ins_cost(INSN_COST * 22);
11391   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11392             "msubw($dst, rscratch1, $src2, $src1" %}
11393 
11394   ins_encode(aarch64_enc_modw(dst, src1, src2));
11395   ins_pipe(idiv_reg_reg);
11396 %}
11397 
11398 // Long Remainder
11399 
11400 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11401   match(Set dst (ModL src1 src2));
11402 
11403   ins_cost(INSN_COST * 38);
11404   format %{ "sdiv   rscratch1, $src1, $src2\n"
11405             "msub($dst, rscratch1, $src2, $src1" %}
11406 
11407   ins_encode(aarch64_enc_mod(dst, src1, src2));
11408   ins_pipe(ldiv_reg_reg);
11409 %}
11410 
11411 // Integer Shifts
11412 
11413 // Shift Left Register
11414 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11415   match(Set dst (LShiftI src1 src2));
11416 
11417   ins_cost(INSN_COST * 2);
11418   format %{ "lslvw  $dst, $src1, $src2" %}
11419 
11420   ins_encode %{
11421     __ lslvw(as_Register($dst$$reg),
11422              as_Register($src1$$reg),
11423              as_Register($src2$$reg));
11424   %}
11425 
11426   ins_pipe(ialu_reg_reg_vshift);
11427 %}
11428 
11429 // Shift Left Immediate
11430 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11431   match(Set dst (LShiftI src1 src2));
11432 
11433   ins_cost(INSN_COST);
11434   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11435 
11436   ins_encode %{
11437     __ lslw(as_Register($dst$$reg),
11438             as_Register($src1$$reg),
11439             $src2$$constant & 0x1f);
11440   %}
11441 
11442   ins_pipe(ialu_reg_shift);
11443 %}
11444 
11445 // Shift Right Logical Register
11446 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11447   match(Set dst (URShiftI src1 src2));
11448 
11449   ins_cost(INSN_COST * 2);
11450   format %{ "lsrvw  $dst, $src1, $src2" %}
11451 
11452   ins_encode %{
11453     __ lsrvw(as_Register($dst$$reg),
11454              as_Register($src1$$reg),
11455              as_Register($src2$$reg));
11456   %}
11457 
11458   ins_pipe(ialu_reg_reg_vshift);
11459 %}
11460 
11461 // Shift Right Logical Immediate
11462 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11463   match(Set dst (URShiftI src1 src2));
11464 
11465   ins_cost(INSN_COST);
11466   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11467 
11468   ins_encode %{
11469     __ lsrw(as_Register($dst$$reg),
11470             as_Register($src1$$reg),
11471             $src2$$constant & 0x1f);
11472   %}
11473 
11474   ins_pipe(ialu_reg_shift);
11475 %}
11476 
11477 // Shift Right Arithmetic Register
11478 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11479   match(Set dst (RShiftI src1 src2));
11480 
11481   ins_cost(INSN_COST * 2);
11482   format %{ "asrvw  $dst, $src1, $src2" %}
11483 
11484   ins_encode %{
11485     __ asrvw(as_Register($dst$$reg),
11486              as_Register($src1$$reg),
11487              as_Register($src2$$reg));
11488   %}
11489 
11490   ins_pipe(ialu_reg_reg_vshift);
11491 %}
11492 
11493 // Shift Right Arithmetic Immediate
11494 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11495   match(Set dst (RShiftI src1 src2));
11496 
11497   ins_cost(INSN_COST);
11498   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11499 
11500   ins_encode %{
11501     __ asrw(as_Register($dst$$reg),
11502             as_Register($src1$$reg),
11503             $src2$$constant & 0x1f);
11504   %}
11505 
11506   ins_pipe(ialu_reg_shift);
11507 %}
11508 
11509 // Combined Int Mask and Right Shift (using UBFM)
11510 // TODO
11511 
11512 // Long Shifts
11513 
11514 // Shift Left Register
11515 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11516   match(Set dst (LShiftL src1 src2));
11517 
11518   ins_cost(INSN_COST * 2);
11519   format %{ "lslv  $dst, $src1, $src2" %}
11520 
11521   ins_encode %{
11522     __ lslv(as_Register($dst$$reg),
11523             as_Register($src1$$reg),
11524             as_Register($src2$$reg));
11525   %}
11526 
11527   ins_pipe(ialu_reg_reg_vshift);
11528 %}
11529 
11530 // Shift Left Immediate
11531 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11532   match(Set dst (LShiftL src1 src2));
11533 
11534   ins_cost(INSN_COST);
11535   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11536 
11537   ins_encode %{
11538     __ lsl(as_Register($dst$$reg),
11539             as_Register($src1$$reg),
11540             $src2$$constant & 0x3f);
11541   %}
11542 
11543   ins_pipe(ialu_reg_shift);
11544 %}
11545 
11546 // Shift Right Logical Register
11547 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11548   match(Set dst (URShiftL src1 src2));
11549 
11550   ins_cost(INSN_COST * 2);
11551   format %{ "lsrv  $dst, $src1, $src2" %}
11552 
11553   ins_encode %{
11554     __ lsrv(as_Register($dst$$reg),
11555             as_Register($src1$$reg),
11556             as_Register($src2$$reg));
11557   %}
11558 
11559   ins_pipe(ialu_reg_reg_vshift);
11560 %}
11561 
11562 // Shift Right Logical Immediate
11563 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11564   match(Set dst (URShiftL src1 src2));
11565 
11566   ins_cost(INSN_COST);
11567   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11568 
11569   ins_encode %{
11570     __ lsr(as_Register($dst$$reg),
11571            as_Register($src1$$reg),
11572            $src2$$constant & 0x3f);
11573   %}
11574 
11575   ins_pipe(ialu_reg_shift);
11576 %}
11577 
11578 // A special-case pattern for card table stores.
11579 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11580   match(Set dst (URShiftL (CastP2X src1) src2));
11581 
11582   ins_cost(INSN_COST);
11583   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11584 
11585   ins_encode %{
11586     __ lsr(as_Register($dst$$reg),
11587            as_Register($src1$$reg),
11588            $src2$$constant & 0x3f);
11589   %}
11590 
11591   ins_pipe(ialu_reg_shift);
11592 %}
11593 
11594 // Shift Right Arithmetic Register
11595 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11596   match(Set dst (RShiftL src1 src2));
11597 
11598   ins_cost(INSN_COST * 2);
11599   format %{ "asrv  $dst, $src1, $src2" %}
11600 
11601   ins_encode %{
11602     __ asrv(as_Register($dst$$reg),
11603             as_Register($src1$$reg),
11604             as_Register($src2$$reg));
11605   %}
11606 
11607   ins_pipe(ialu_reg_reg_vshift);
11608 %}
11609 
11610 // Shift Right Arithmetic Immediate
11611 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11612   match(Set dst (RShiftL src1 src2));
11613 
11614   ins_cost(INSN_COST);
11615   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11616 
11617   ins_encode %{
11618     __ asr(as_Register($dst$$reg),
11619            as_Register($src1$$reg),
11620            $src2$$constant & 0x3f);
11621   %}
11622 
11623   ins_pipe(ialu_reg_shift);
11624 %}
11625 
11626 // BEGIN This section of the file is automatically generated. Do not edit --------------
11627 // This section is generated from aarch64_ad.m4
11628 
11629 
11630 // This pattern is automatically generated from aarch64_ad.m4
11631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11632 instruct regL_not_reg(iRegLNoSp dst,
11633                          iRegL src1, immL_M1 m1,
11634                          rFlagsReg cr) %{
11635   match(Set dst (XorL src1 m1));
11636   ins_cost(INSN_COST);
11637   format %{ "eon  $dst, $src1, zr" %}
11638 
11639   ins_encode %{
11640     __ eon(as_Register($dst$$reg),
11641               as_Register($src1$$reg),
11642               zr,
11643               Assembler::LSL, 0);
11644   %}
11645 
11646   ins_pipe(ialu_reg);
11647 %}
11648 
11649 // This pattern is automatically generated from aarch64_ad.m4
11650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11651 instruct regI_not_reg(iRegINoSp dst,
11652                          iRegIorL2I src1, immI_M1 m1,
11653                          rFlagsReg cr) %{
11654   match(Set dst (XorI src1 m1));
11655   ins_cost(INSN_COST);
11656   format %{ "eonw  $dst, $src1, zr" %}
11657 
11658   ins_encode %{
11659     __ eonw(as_Register($dst$$reg),
11660               as_Register($src1$$reg),
11661               zr,
11662               Assembler::LSL, 0);
11663   %}
11664 
11665   ins_pipe(ialu_reg);
11666 %}
11667 
11668 // This pattern is automatically generated from aarch64_ad.m4
11669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11670 instruct AndI_reg_not_reg(iRegINoSp dst,
11671                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11672   match(Set dst (AndI src1 (XorI src2 m1)));
11673   ins_cost(INSN_COST);
11674   format %{ "bicw  $dst, $src1, $src2" %}
11675 
11676   ins_encode %{
11677     __ bicw(as_Register($dst$$reg),
11678               as_Register($src1$$reg),
11679               as_Register($src2$$reg),
11680               Assembler::LSL, 0);
11681   %}
11682 
11683   ins_pipe(ialu_reg_reg);
11684 %}
11685 
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688 instruct AndL_reg_not_reg(iRegLNoSp dst,
11689                          iRegL src1, iRegL src2, immL_M1 m1) %{
11690   match(Set dst (AndL src1 (XorL src2 m1)));
11691   ins_cost(INSN_COST);
11692   format %{ "bic  $dst, $src1, $src2" %}
11693 
11694   ins_encode %{
11695     __ bic(as_Register($dst$$reg),
11696               as_Register($src1$$reg),
11697               as_Register($src2$$reg),
11698               Assembler::LSL, 0);
11699   %}
11700 
11701   ins_pipe(ialu_reg_reg);
11702 %}
11703 
11704 // This pattern is automatically generated from aarch64_ad.m4
11705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11706 instruct OrI_reg_not_reg(iRegINoSp dst,
11707                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11708   match(Set dst (OrI src1 (XorI src2 m1)));
11709   ins_cost(INSN_COST);
11710   format %{ "ornw  $dst, $src1, $src2" %}
11711 
11712   ins_encode %{
11713     __ ornw(as_Register($dst$$reg),
11714               as_Register($src1$$reg),
11715               as_Register($src2$$reg),
11716               Assembler::LSL, 0);
11717   %}
11718 
11719   ins_pipe(ialu_reg_reg);
11720 %}
11721 
11722 // This pattern is automatically generated from aarch64_ad.m4
11723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11724 instruct OrL_reg_not_reg(iRegLNoSp dst,
11725                          iRegL src1, iRegL src2, immL_M1 m1) %{
11726   match(Set dst (OrL src1 (XorL src2 m1)));
11727   ins_cost(INSN_COST);
11728   format %{ "orn  $dst, $src1, $src2" %}
11729 
11730   ins_encode %{
11731     __ orn(as_Register($dst$$reg),
11732               as_Register($src1$$reg),
11733               as_Register($src2$$reg),
11734               Assembler::LSL, 0);
11735   %}
11736 
11737   ins_pipe(ialu_reg_reg);
11738 %}
11739 
11740 // This pattern is automatically generated from aarch64_ad.m4
11741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11742 instruct XorI_reg_not_reg(iRegINoSp dst,
11743                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11744   match(Set dst (XorI m1 (XorI src2 src1)));
11745   ins_cost(INSN_COST);
11746   format %{ "eonw  $dst, $src1, $src2" %}
11747 
11748   ins_encode %{
11749     __ eonw(as_Register($dst$$reg),
11750               as_Register($src1$$reg),
11751               as_Register($src2$$reg),
11752               Assembler::LSL, 0);
11753   %}
11754 
11755   ins_pipe(ialu_reg_reg);
11756 %}
11757 
11758 // This pattern is automatically generated from aarch64_ad.m4
11759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11760 instruct XorL_reg_not_reg(iRegLNoSp dst,
11761                          iRegL src1, iRegL src2, immL_M1 m1) %{
11762   match(Set dst (XorL m1 (XorL src2 src1)));
11763   ins_cost(INSN_COST);
11764   format %{ "eon  $dst, $src1, $src2" %}
11765 
11766   ins_encode %{
11767     __ eon(as_Register($dst$$reg),
11768               as_Register($src1$$reg),
11769               as_Register($src2$$reg),
11770               Assembler::LSL, 0);
11771   %}
11772 
11773   ins_pipe(ialu_reg_reg);
11774 %}
11775 
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 // val & (-1 ^ (val >>> shift)) ==> bicw
11779 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11780                          iRegIorL2I src1, iRegIorL2I src2,
11781                          immI src3, immI_M1 src4) %{
11782   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11783   ins_cost(1.9 * INSN_COST);
11784   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11785 
11786   ins_encode %{
11787     __ bicw(as_Register($dst$$reg),
11788               as_Register($src1$$reg),
11789               as_Register($src2$$reg),
11790               Assembler::LSR,
11791               $src3$$constant & 0x1f);
11792   %}
11793 
11794   ins_pipe(ialu_reg_reg_shift);
11795 %}
11796 
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 // val & (-1 ^ (val >>> shift)) ==> bic
11800 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11801                          iRegL src1, iRegL src2,
11802                          immI src3, immL_M1 src4) %{
11803   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11804   ins_cost(1.9 * INSN_COST);
11805   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11806 
11807   ins_encode %{
11808     __ bic(as_Register($dst$$reg),
11809               as_Register($src1$$reg),
11810               as_Register($src2$$reg),
11811               Assembler::LSR,
11812               $src3$$constant & 0x3f);
11813   %}
11814 
11815   ins_pipe(ialu_reg_reg_shift);
11816 %}
11817 
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 // val & (-1 ^ (val >> shift)) ==> bicw
11821 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11822                          iRegIorL2I src1, iRegIorL2I src2,
11823                          immI src3, immI_M1 src4) %{
11824   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11825   ins_cost(1.9 * INSN_COST);
11826   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11827 
11828   ins_encode %{
11829     __ bicw(as_Register($dst$$reg),
11830               as_Register($src1$$reg),
11831               as_Register($src2$$reg),
11832               Assembler::ASR,
11833               $src3$$constant & 0x1f);
11834   %}
11835 
11836   ins_pipe(ialu_reg_reg_shift);
11837 %}
11838 
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 // val & (-1 ^ (val >> shift)) ==> bic
11842 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11843                          iRegL src1, iRegL src2,
11844                          immI src3, immL_M1 src4) %{
11845   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11846   ins_cost(1.9 * INSN_COST);
11847   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11848 
11849   ins_encode %{
11850     __ bic(as_Register($dst$$reg),
11851               as_Register($src1$$reg),
11852               as_Register($src2$$reg),
11853               Assembler::ASR,
11854               $src3$$constant & 0x3f);
11855   %}
11856 
11857   ins_pipe(ialu_reg_reg_shift);
11858 %}
11859 
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 // val & (-1 ^ (val ror shift)) ==> bicw
11863 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11864                          iRegIorL2I src1, iRegIorL2I src2,
11865                          immI src3, immI_M1 src4) %{
11866   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11867   ins_cost(1.9 * INSN_COST);
11868   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11869 
11870   ins_encode %{
11871     __ bicw(as_Register($dst$$reg),
11872               as_Register($src1$$reg),
11873               as_Register($src2$$reg),
11874               Assembler::ROR,
11875               $src3$$constant & 0x1f);
11876   %}
11877 
11878   ins_pipe(ialu_reg_reg_shift);
11879 %}
11880 
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 // val & (-1 ^ (val ror shift)) ==> bic
11884 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11885                          iRegL src1, iRegL src2,
11886                          immI src3, immL_M1 src4) %{
11887   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11888   ins_cost(1.9 * INSN_COST);
11889   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11890 
11891   ins_encode %{
11892     __ bic(as_Register($dst$$reg),
11893               as_Register($src1$$reg),
11894               as_Register($src2$$reg),
11895               Assembler::ROR,
11896               $src3$$constant & 0x3f);
11897   %}
11898 
11899   ins_pipe(ialu_reg_reg_shift);
11900 %}
11901 
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 // val & (-1 ^ (val << shift)) ==> bicw
11905 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11906                          iRegIorL2I src1, iRegIorL2I src2,
11907                          immI src3, immI_M1 src4) %{
11908   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11909   ins_cost(1.9 * INSN_COST);
11910   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11911 
11912   ins_encode %{
11913     __ bicw(as_Register($dst$$reg),
11914               as_Register($src1$$reg),
11915               as_Register($src2$$reg),
11916               Assembler::LSL,
11917               $src3$$constant & 0x1f);
11918   %}
11919 
11920   ins_pipe(ialu_reg_reg_shift);
11921 %}
11922 
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 // val & (-1 ^ (val << shift)) ==> bic
11926 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11927                          iRegL src1, iRegL src2,
11928                          immI src3, immL_M1 src4) %{
11929   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11930   ins_cost(1.9 * INSN_COST);
11931   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11932 
11933   ins_encode %{
11934     __ bic(as_Register($dst$$reg),
11935               as_Register($src1$$reg),
11936               as_Register($src2$$reg),
11937               Assembler::LSL,
11938               $src3$$constant & 0x3f);
11939   %}
11940 
11941   ins_pipe(ialu_reg_reg_shift);
11942 %}
11943 
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11947 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11948                          iRegIorL2I src1, iRegIorL2I src2,
11949                          immI src3, immI_M1 src4) %{
11950   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11951   ins_cost(1.9 * INSN_COST);
11952   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11953 
11954   ins_encode %{
11955     __ eonw(as_Register($dst$$reg),
11956               as_Register($src1$$reg),
11957               as_Register($src2$$reg),
11958               Assembler::LSR,
11959               $src3$$constant & 0x1f);
11960   %}
11961 
11962   ins_pipe(ialu_reg_reg_shift);
11963 %}
11964 
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 // val ^ (-1 ^ (val >>> shift)) ==> eon
11968 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11969                          iRegL src1, iRegL src2,
11970                          immI src3, immL_M1 src4) %{
11971   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11972   ins_cost(1.9 * INSN_COST);
11973   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11974 
11975   ins_encode %{
11976     __ eon(as_Register($dst$$reg),
11977               as_Register($src1$$reg),
11978               as_Register($src2$$reg),
11979               Assembler::LSR,
11980               $src3$$constant & 0x3f);
11981   %}
11982 
11983   ins_pipe(ialu_reg_reg_shift);
11984 %}
11985 
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 // val ^ (-1 ^ (val >> shift)) ==> eonw
11989 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11990                          iRegIorL2I src1, iRegIorL2I src2,
11991                          immI src3, immI_M1 src4) %{
11992   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11993   ins_cost(1.9 * INSN_COST);
11994   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11995 
11996   ins_encode %{
11997     __ eonw(as_Register($dst$$reg),
11998               as_Register($src1$$reg),
11999               as_Register($src2$$reg),
12000               Assembler::ASR,
12001               $src3$$constant & 0x1f);
12002   %}
12003 
12004   ins_pipe(ialu_reg_reg_shift);
12005 %}
12006 
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 // val ^ (-1 ^ (val >> shift)) ==> eon
12010 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
12011                          iRegL src1, iRegL src2,
12012                          immI src3, immL_M1 src4) %{
12013   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
12014   ins_cost(1.9 * INSN_COST);
12015   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
12016 
12017   ins_encode %{
12018     __ eon(as_Register($dst$$reg),
12019               as_Register($src1$$reg),
12020               as_Register($src2$$reg),
12021               Assembler::ASR,
12022               $src3$$constant & 0x3f);
12023   %}
12024 
12025   ins_pipe(ialu_reg_reg_shift);
12026 %}
12027 
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 // val ^ (-1 ^ (val ror shift)) ==> eonw
12031 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
12032                          iRegIorL2I src1, iRegIorL2I src2,
12033                          immI src3, immI_M1 src4) %{
12034   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
12035   ins_cost(1.9 * INSN_COST);
12036   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
12037 
12038   ins_encode %{
12039     __ eonw(as_Register($dst$$reg),
12040               as_Register($src1$$reg),
12041               as_Register($src2$$reg),
12042               Assembler::ROR,
12043               $src3$$constant & 0x1f);
12044   %}
12045 
12046   ins_pipe(ialu_reg_reg_shift);
12047 %}
12048 
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 // val ^ (-1 ^ (val ror shift)) ==> eon
12052 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
12053                          iRegL src1, iRegL src2,
12054                          immI src3, immL_M1 src4) %{
12055   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
12056   ins_cost(1.9 * INSN_COST);
12057   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
12058 
12059   ins_encode %{
12060     __ eon(as_Register($dst$$reg),
12061               as_Register($src1$$reg),
12062               as_Register($src2$$reg),
12063               Assembler::ROR,
12064               $src3$$constant & 0x3f);
12065   %}
12066 
12067   ins_pipe(ialu_reg_reg_shift);
12068 %}
12069 
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 // val ^ (-1 ^ (val << shift)) ==> eonw
12073 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
12074                          iRegIorL2I src1, iRegIorL2I src2,
12075                          immI src3, immI_M1 src4) %{
12076   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
12077   ins_cost(1.9 * INSN_COST);
12078   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
12079 
12080   ins_encode %{
12081     __ eonw(as_Register($dst$$reg),
12082               as_Register($src1$$reg),
12083               as_Register($src2$$reg),
12084               Assembler::LSL,
12085               $src3$$constant & 0x1f);
12086   %}
12087 
12088   ins_pipe(ialu_reg_reg_shift);
12089 %}
12090 
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 // val ^ (-1 ^ (val << shift)) ==> eon
12094 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
12095                          iRegL src1, iRegL src2,
12096                          immI src3, immL_M1 src4) %{
12097   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12098   ins_cost(1.9 * INSN_COST);
12099   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
12100 
12101   ins_encode %{
12102     __ eon(as_Register($dst$$reg),
12103               as_Register($src1$$reg),
12104               as_Register($src2$$reg),
12105               Assembler::LSL,
12106               $src3$$constant & 0x3f);
12107   %}
12108 
12109   ins_pipe(ialu_reg_reg_shift);
12110 %}
12111 
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 // val | (-1 ^ (val >>> shift)) ==> ornw
12115 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12116                          iRegIorL2I src1, iRegIorL2I src2,
12117                          immI src3, immI_M1 src4) %{
12118   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12119   ins_cost(1.9 * INSN_COST);
12120   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
12121 
12122   ins_encode %{
12123     __ ornw(as_Register($dst$$reg),
12124               as_Register($src1$$reg),
12125               as_Register($src2$$reg),
12126               Assembler::LSR,
12127               $src3$$constant & 0x1f);
12128   %}
12129 
12130   ins_pipe(ialu_reg_reg_shift);
12131 %}
12132 
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 // val | (-1 ^ (val >>> shift)) ==> orn
12136 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12137                          iRegL src1, iRegL src2,
12138                          immI src3, immL_M1 src4) %{
12139   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12140   ins_cost(1.9 * INSN_COST);
12141   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
12142 
12143   ins_encode %{
12144     __ orn(as_Register($dst$$reg),
12145               as_Register($src1$$reg),
12146               as_Register($src2$$reg),
12147               Assembler::LSR,
12148               $src3$$constant & 0x3f);
12149   %}
12150 
12151   ins_pipe(ialu_reg_reg_shift);
12152 %}
12153 
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 // val | (-1 ^ (val >> shift)) ==> ornw
12157 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12158                          iRegIorL2I src1, iRegIorL2I src2,
12159                          immI src3, immI_M1 src4) %{
12160   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12161   ins_cost(1.9 * INSN_COST);
12162   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12163 
12164   ins_encode %{
12165     __ ornw(as_Register($dst$$reg),
12166               as_Register($src1$$reg),
12167               as_Register($src2$$reg),
12168               Assembler::ASR,
12169               $src3$$constant & 0x1f);
12170   %}
12171 
12172   ins_pipe(ialu_reg_reg_shift);
12173 %}
12174 
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 // val | (-1 ^ (val >> shift)) ==> orn
12178 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12179                          iRegL src1, iRegL src2,
12180                          immI src3, immL_M1 src4) %{
12181   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12182   ins_cost(1.9 * INSN_COST);
12183   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12184 
12185   ins_encode %{
12186     __ orn(as_Register($dst$$reg),
12187               as_Register($src1$$reg),
12188               as_Register($src2$$reg),
12189               Assembler::ASR,
12190               $src3$$constant & 0x3f);
12191   %}
12192 
12193   ins_pipe(ialu_reg_reg_shift);
12194 %}
12195 
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 // val | (-1 ^ (val ror shift)) ==> ornw
12199 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12200                          iRegIorL2I src1, iRegIorL2I src2,
12201                          immI src3, immI_M1 src4) %{
12202   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12203   ins_cost(1.9 * INSN_COST);
12204   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12205 
12206   ins_encode %{
12207     __ ornw(as_Register($dst$$reg),
12208               as_Register($src1$$reg),
12209               as_Register($src2$$reg),
12210               Assembler::ROR,
12211               $src3$$constant & 0x1f);
12212   %}
12213 
12214   ins_pipe(ialu_reg_reg_shift);
12215 %}
12216 
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 // val | (-1 ^ (val ror shift)) ==> orn
12220 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12221                          iRegL src1, iRegL src2,
12222                          immI src3, immL_M1 src4) %{
12223   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12224   ins_cost(1.9 * INSN_COST);
12225   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12226 
12227   ins_encode %{
12228     __ orn(as_Register($dst$$reg),
12229               as_Register($src1$$reg),
12230               as_Register($src2$$reg),
12231               Assembler::ROR,
12232               $src3$$constant & 0x3f);
12233   %}
12234 
12235   ins_pipe(ialu_reg_reg_shift);
12236 %}
12237 
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 // val | (-1 ^ (val << shift)) ==> ornw
12241 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12242                          iRegIorL2I src1, iRegIorL2I src2,
12243                          immI src3, immI_M1 src4) %{
12244   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12245   ins_cost(1.9 * INSN_COST);
12246   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12247 
12248   ins_encode %{
12249     __ ornw(as_Register($dst$$reg),
12250               as_Register($src1$$reg),
12251               as_Register($src2$$reg),
12252               Assembler::LSL,
12253               $src3$$constant & 0x1f);
12254   %}
12255 
12256   ins_pipe(ialu_reg_reg_shift);
12257 %}
12258 
12259 // This pattern is automatically generated from aarch64_ad.m4
12260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12261 // val | (-1 ^ (val << shift)) ==> orn
12262 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12263                          iRegL src1, iRegL src2,
12264                          immI src3, immL_M1 src4) %{
12265   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12266   ins_cost(1.9 * INSN_COST);
12267   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12268 
12269   ins_encode %{
12270     __ orn(as_Register($dst$$reg),
12271               as_Register($src1$$reg),
12272               as_Register($src2$$reg),
12273               Assembler::LSL,
12274               $src3$$constant & 0x3f);
12275   %}
12276 
12277   ins_pipe(ialu_reg_reg_shift);
12278 %}
12279 
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12283                          iRegIorL2I src1, iRegIorL2I src2,
12284                          immI src3) %{
12285   match(Set dst (AndI src1 (URShiftI src2 src3)));
12286 
12287   ins_cost(1.9 * INSN_COST);
12288   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12289 
12290   ins_encode %{
12291     __ andw(as_Register($dst$$reg),
12292               as_Register($src1$$reg),
12293               as_Register($src2$$reg),
12294               Assembler::LSR,
12295               $src3$$constant & 0x1f);
12296   %}
12297 
12298   ins_pipe(ialu_reg_reg_shift);
12299 %}
12300 
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12304                          iRegL src1, iRegL src2,
12305                          immI src3) %{
12306   match(Set dst (AndL src1 (URShiftL src2 src3)));
12307 
12308   ins_cost(1.9 * INSN_COST);
12309   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12310 
12311   ins_encode %{
12312     __ andr(as_Register($dst$$reg),
12313               as_Register($src1$$reg),
12314               as_Register($src2$$reg),
12315               Assembler::LSR,
12316               $src3$$constant & 0x3f);
12317   %}
12318 
12319   ins_pipe(ialu_reg_reg_shift);
12320 %}
12321 
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12325                          iRegIorL2I src1, iRegIorL2I src2,
12326                          immI src3) %{
12327   match(Set dst (AndI src1 (RShiftI src2 src3)));
12328 
12329   ins_cost(1.9 * INSN_COST);
12330   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12331 
12332   ins_encode %{
12333     __ andw(as_Register($dst$$reg),
12334               as_Register($src1$$reg),
12335               as_Register($src2$$reg),
12336               Assembler::ASR,
12337               $src3$$constant & 0x1f);
12338   %}
12339 
12340   ins_pipe(ialu_reg_reg_shift);
12341 %}
12342 
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12346                          iRegL src1, iRegL src2,
12347                          immI src3) %{
12348   match(Set dst (AndL src1 (RShiftL src2 src3)));
12349 
12350   ins_cost(1.9 * INSN_COST);
12351   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12352 
12353   ins_encode %{
12354     __ andr(as_Register($dst$$reg),
12355               as_Register($src1$$reg),
12356               as_Register($src2$$reg),
12357               Assembler::ASR,
12358               $src3$$constant & 0x3f);
12359   %}
12360 
12361   ins_pipe(ialu_reg_reg_shift);
12362 %}
12363 
12364 // This pattern is automatically generated from aarch64_ad.m4
12365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12366 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12367                          iRegIorL2I src1, iRegIorL2I src2,
12368                          immI src3) %{
12369   match(Set dst (AndI src1 (LShiftI src2 src3)));
12370 
12371   ins_cost(1.9 * INSN_COST);
12372   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12373 
12374   ins_encode %{
12375     __ andw(as_Register($dst$$reg),
12376               as_Register($src1$$reg),
12377               as_Register($src2$$reg),
12378               Assembler::LSL,
12379               $src3$$constant & 0x1f);
12380   %}
12381 
12382   ins_pipe(ialu_reg_reg_shift);
12383 %}
12384 
12385 // This pattern is automatically generated from aarch64_ad.m4
12386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12387 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12388                          iRegL src1, iRegL src2,
12389                          immI src3) %{
12390   match(Set dst (AndL src1 (LShiftL src2 src3)));
12391 
12392   ins_cost(1.9 * INSN_COST);
12393   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12394 
12395   ins_encode %{
12396     __ andr(as_Register($dst$$reg),
12397               as_Register($src1$$reg),
12398               as_Register($src2$$reg),
12399               Assembler::LSL,
12400               $src3$$constant & 0x3f);
12401   %}
12402 
12403   ins_pipe(ialu_reg_reg_shift);
12404 %}
12405 
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12409                          iRegIorL2I src1, iRegIorL2I src2,
12410                          immI src3) %{
12411   match(Set dst (AndI src1 (RotateRight src2 src3)));
12412 
12413   ins_cost(1.9 * INSN_COST);
12414   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12415 
12416   ins_encode %{
12417     __ andw(as_Register($dst$$reg),
12418               as_Register($src1$$reg),
12419               as_Register($src2$$reg),
12420               Assembler::ROR,
12421               $src3$$constant & 0x1f);
12422   %}
12423 
12424   ins_pipe(ialu_reg_reg_shift);
12425 %}
12426 
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12430                          iRegL src1, iRegL src2,
12431                          immI src3) %{
12432   match(Set dst (AndL src1 (RotateRight src2 src3)));
12433 
12434   ins_cost(1.9 * INSN_COST);
12435   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12436 
12437   ins_encode %{
12438     __ andr(as_Register($dst$$reg),
12439               as_Register($src1$$reg),
12440               as_Register($src2$$reg),
12441               Assembler::ROR,
12442               $src3$$constant & 0x3f);
12443   %}
12444 
12445   ins_pipe(ialu_reg_reg_shift);
12446 %}
12447 
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12451                          iRegIorL2I src1, iRegIorL2I src2,
12452                          immI src3) %{
12453   match(Set dst (XorI src1 (URShiftI src2 src3)));
12454 
12455   ins_cost(1.9 * INSN_COST);
12456   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12457 
12458   ins_encode %{
12459     __ eorw(as_Register($dst$$reg),
12460               as_Register($src1$$reg),
12461               as_Register($src2$$reg),
12462               Assembler::LSR,
12463               $src3$$constant & 0x1f);
12464   %}
12465 
12466   ins_pipe(ialu_reg_reg_shift);
12467 %}
12468 
12469 // This pattern is automatically generated from aarch64_ad.m4
12470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12471 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12472                          iRegL src1, iRegL src2,
12473                          immI src3) %{
12474   match(Set dst (XorL src1 (URShiftL src2 src3)));
12475 
12476   ins_cost(1.9 * INSN_COST);
12477   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12478 
12479   ins_encode %{
12480     __ eor(as_Register($dst$$reg),
12481               as_Register($src1$$reg),
12482               as_Register($src2$$reg),
12483               Assembler::LSR,
12484               $src3$$constant & 0x3f);
12485   %}
12486 
12487   ins_pipe(ialu_reg_reg_shift);
12488 %}
12489 
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12493                          iRegIorL2I src1, iRegIorL2I src2,
12494                          immI src3) %{
12495   match(Set dst (XorI src1 (RShiftI src2 src3)));
12496 
12497   ins_cost(1.9 * INSN_COST);
12498   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12499 
12500   ins_encode %{
12501     __ eorw(as_Register($dst$$reg),
12502               as_Register($src1$$reg),
12503               as_Register($src2$$reg),
12504               Assembler::ASR,
12505               $src3$$constant & 0x1f);
12506   %}
12507 
12508   ins_pipe(ialu_reg_reg_shift);
12509 %}
12510 
12511 // This pattern is automatically generated from aarch64_ad.m4
12512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12513 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12514                          iRegL src1, iRegL src2,
12515                          immI src3) %{
12516   match(Set dst (XorL src1 (RShiftL src2 src3)));
12517 
12518   ins_cost(1.9 * INSN_COST);
12519   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12520 
12521   ins_encode %{
12522     __ eor(as_Register($dst$$reg),
12523               as_Register($src1$$reg),
12524               as_Register($src2$$reg),
12525               Assembler::ASR,
12526               $src3$$constant & 0x3f);
12527   %}
12528 
12529   ins_pipe(ialu_reg_reg_shift);
12530 %}
12531 
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12535                          iRegIorL2I src1, iRegIorL2I src2,
12536                          immI src3) %{
12537   match(Set dst (XorI src1 (LShiftI src2 src3)));
12538 
12539   ins_cost(1.9 * INSN_COST);
12540   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12541 
12542   ins_encode %{
12543     __ eorw(as_Register($dst$$reg),
12544               as_Register($src1$$reg),
12545               as_Register($src2$$reg),
12546               Assembler::LSL,
12547               $src3$$constant & 0x1f);
12548   %}
12549 
12550   ins_pipe(ialu_reg_reg_shift);
12551 %}
12552 
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12556                          iRegL src1, iRegL src2,
12557                          immI src3) %{
12558   match(Set dst (XorL src1 (LShiftL src2 src3)));
12559 
12560   ins_cost(1.9 * INSN_COST);
12561   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12562 
12563   ins_encode %{
12564     __ eor(as_Register($dst$$reg),
12565               as_Register($src1$$reg),
12566               as_Register($src2$$reg),
12567               Assembler::LSL,
12568               $src3$$constant & 0x3f);
12569   %}
12570 
12571   ins_pipe(ialu_reg_reg_shift);
12572 %}
12573 
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12577                          iRegIorL2I src1, iRegIorL2I src2,
12578                          immI src3) %{
12579   match(Set dst (XorI src1 (RotateRight src2 src3)));
12580 
12581   ins_cost(1.9 * INSN_COST);
12582   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12583 
12584   ins_encode %{
12585     __ eorw(as_Register($dst$$reg),
12586               as_Register($src1$$reg),
12587               as_Register($src2$$reg),
12588               Assembler::ROR,
12589               $src3$$constant & 0x1f);
12590   %}
12591 
12592   ins_pipe(ialu_reg_reg_shift);
12593 %}
12594 
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12598                          iRegL src1, iRegL src2,
12599                          immI src3) %{
12600   match(Set dst (XorL src1 (RotateRight src2 src3)));
12601 
12602   ins_cost(1.9 * INSN_COST);
12603   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12604 
12605   ins_encode %{
12606     __ eor(as_Register($dst$$reg),
12607               as_Register($src1$$reg),
12608               as_Register($src2$$reg),
12609               Assembler::ROR,
12610               $src3$$constant & 0x3f);
12611   %}
12612 
12613   ins_pipe(ialu_reg_reg_shift);
12614 %}
12615 
12616 // This pattern is automatically generated from aarch64_ad.m4
12617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12618 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12619                          iRegIorL2I src1, iRegIorL2I src2,
12620                          immI src3) %{
12621   match(Set dst (OrI src1 (URShiftI src2 src3)));
12622 
12623   ins_cost(1.9 * INSN_COST);
12624   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12625 
12626   ins_encode %{
12627     __ orrw(as_Register($dst$$reg),
12628               as_Register($src1$$reg),
12629               as_Register($src2$$reg),
12630               Assembler::LSR,
12631               $src3$$constant & 0x1f);
12632   %}
12633 
12634   ins_pipe(ialu_reg_reg_shift);
12635 %}
12636 
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12640                          iRegL src1, iRegL src2,
12641                          immI src3) %{
12642   match(Set dst (OrL src1 (URShiftL src2 src3)));
12643 
12644   ins_cost(1.9 * INSN_COST);
12645   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12646 
12647   ins_encode %{
12648     __ orr(as_Register($dst$$reg),
12649               as_Register($src1$$reg),
12650               as_Register($src2$$reg),
12651               Assembler::LSR,
12652               $src3$$constant & 0x3f);
12653   %}
12654 
12655   ins_pipe(ialu_reg_reg_shift);
12656 %}
12657 
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12661                          iRegIorL2I src1, iRegIorL2I src2,
12662                          immI src3) %{
12663   match(Set dst (OrI src1 (RShiftI src2 src3)));
12664 
12665   ins_cost(1.9 * INSN_COST);
12666   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12667 
12668   ins_encode %{
12669     __ orrw(as_Register($dst$$reg),
12670               as_Register($src1$$reg),
12671               as_Register($src2$$reg),
12672               Assembler::ASR,
12673               $src3$$constant & 0x1f);
12674   %}
12675 
12676   ins_pipe(ialu_reg_reg_shift);
12677 %}
12678 
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12682                          iRegL src1, iRegL src2,
12683                          immI src3) %{
12684   match(Set dst (OrL src1 (RShiftL src2 src3)));
12685 
12686   ins_cost(1.9 * INSN_COST);
12687   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12688 
12689   ins_encode %{
12690     __ orr(as_Register($dst$$reg),
12691               as_Register($src1$$reg),
12692               as_Register($src2$$reg),
12693               Assembler::ASR,
12694               $src3$$constant & 0x3f);
12695   %}
12696 
12697   ins_pipe(ialu_reg_reg_shift);
12698 %}
12699 
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12703                          iRegIorL2I src1, iRegIorL2I src2,
12704                          immI src3) %{
12705   match(Set dst (OrI src1 (LShiftI src2 src3)));
12706 
12707   ins_cost(1.9 * INSN_COST);
12708   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12709 
12710   ins_encode %{
12711     __ orrw(as_Register($dst$$reg),
12712               as_Register($src1$$reg),
12713               as_Register($src2$$reg),
12714               Assembler::LSL,
12715               $src3$$constant & 0x1f);
12716   %}
12717 
12718   ins_pipe(ialu_reg_reg_shift);
12719 %}
12720 
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12724                          iRegL src1, iRegL src2,
12725                          immI src3) %{
12726   match(Set dst (OrL src1 (LShiftL src2 src3)));
12727 
12728   ins_cost(1.9 * INSN_COST);
12729   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12730 
12731   ins_encode %{
12732     __ orr(as_Register($dst$$reg),
12733               as_Register($src1$$reg),
12734               as_Register($src2$$reg),
12735               Assembler::LSL,
12736               $src3$$constant & 0x3f);
12737   %}
12738 
12739   ins_pipe(ialu_reg_reg_shift);
12740 %}
12741 
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12745                          iRegIorL2I src1, iRegIorL2I src2,
12746                          immI src3) %{
12747   match(Set dst (OrI src1 (RotateRight src2 src3)));
12748 
12749   ins_cost(1.9 * INSN_COST);
12750   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12751 
12752   ins_encode %{
12753     __ orrw(as_Register($dst$$reg),
12754               as_Register($src1$$reg),
12755               as_Register($src2$$reg),
12756               Assembler::ROR,
12757               $src3$$constant & 0x1f);
12758   %}
12759 
12760   ins_pipe(ialu_reg_reg_shift);
12761 %}
12762 
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12766                          iRegL src1, iRegL src2,
12767                          immI src3) %{
12768   match(Set dst (OrL src1 (RotateRight src2 src3)));
12769 
12770   ins_cost(1.9 * INSN_COST);
12771   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12772 
12773   ins_encode %{
12774     __ orr(as_Register($dst$$reg),
12775               as_Register($src1$$reg),
12776               as_Register($src2$$reg),
12777               Assembler::ROR,
12778               $src3$$constant & 0x3f);
12779   %}
12780 
12781   ins_pipe(ialu_reg_reg_shift);
12782 %}
12783 
12784 // This pattern is automatically generated from aarch64_ad.m4
12785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12786 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12787                          iRegIorL2I src1, iRegIorL2I src2,
12788                          immI src3) %{
12789   match(Set dst (AddI src1 (URShiftI src2 src3)));
12790 
12791   ins_cost(1.9 * INSN_COST);
12792   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12793 
12794   ins_encode %{
12795     __ addw(as_Register($dst$$reg),
12796               as_Register($src1$$reg),
12797               as_Register($src2$$reg),
12798               Assembler::LSR,
12799               $src3$$constant & 0x1f);
12800   %}
12801 
12802   ins_pipe(ialu_reg_reg_shift);
12803 %}
12804 
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12808                          iRegL src1, iRegL src2,
12809                          immI src3) %{
12810   match(Set dst (AddL src1 (URShiftL src2 src3)));
12811 
12812   ins_cost(1.9 * INSN_COST);
12813   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12814 
12815   ins_encode %{
12816     __ add(as_Register($dst$$reg),
12817               as_Register($src1$$reg),
12818               as_Register($src2$$reg),
12819               Assembler::LSR,
12820               $src3$$constant & 0x3f);
12821   %}
12822 
12823   ins_pipe(ialu_reg_reg_shift);
12824 %}
12825 
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12829                          iRegIorL2I src1, iRegIorL2I src2,
12830                          immI src3) %{
12831   match(Set dst (AddI src1 (RShiftI src2 src3)));
12832 
12833   ins_cost(1.9 * INSN_COST);
12834   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12835 
12836   ins_encode %{
12837     __ addw(as_Register($dst$$reg),
12838               as_Register($src1$$reg),
12839               as_Register($src2$$reg),
12840               Assembler::ASR,
12841               $src3$$constant & 0x1f);
12842   %}
12843 
12844   ins_pipe(ialu_reg_reg_shift);
12845 %}
12846 
12847 // This pattern is automatically generated from aarch64_ad.m4
12848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12849 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12850                          iRegL src1, iRegL src2,
12851                          immI src3) %{
12852   match(Set dst (AddL src1 (RShiftL src2 src3)));
12853 
12854   ins_cost(1.9 * INSN_COST);
12855   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12856 
12857   ins_encode %{
12858     __ add(as_Register($dst$$reg),
12859               as_Register($src1$$reg),
12860               as_Register($src2$$reg),
12861               Assembler::ASR,
12862               $src3$$constant & 0x3f);
12863   %}
12864 
12865   ins_pipe(ialu_reg_reg_shift);
12866 %}
12867 
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12871                          iRegIorL2I src1, iRegIorL2I src2,
12872                          immI src3) %{
12873   match(Set dst (AddI src1 (LShiftI src2 src3)));
12874 
12875   ins_cost(1.9 * INSN_COST);
12876   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12877 
12878   ins_encode %{
12879     __ addw(as_Register($dst$$reg),
12880               as_Register($src1$$reg),
12881               as_Register($src2$$reg),
12882               Assembler::LSL,
12883               $src3$$constant & 0x1f);
12884   %}
12885 
12886   ins_pipe(ialu_reg_reg_shift);
12887 %}
12888 
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12892                          iRegL src1, iRegL src2,
12893                          immI src3) %{
12894   match(Set dst (AddL src1 (LShiftL src2 src3)));
12895 
12896   ins_cost(1.9 * INSN_COST);
12897   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12898 
12899   ins_encode %{
12900     __ add(as_Register($dst$$reg),
12901               as_Register($src1$$reg),
12902               as_Register($src2$$reg),
12903               Assembler::LSL,
12904               $src3$$constant & 0x3f);
12905   %}
12906 
12907   ins_pipe(ialu_reg_reg_shift);
12908 %}
12909 
12910 // This pattern is automatically generated from aarch64_ad.m4
12911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12912 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12913                          iRegIorL2I src1, iRegIorL2I src2,
12914                          immI src3) %{
12915   match(Set dst (SubI src1 (URShiftI src2 src3)));
12916 
12917   ins_cost(1.9 * INSN_COST);
12918   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12919 
12920   ins_encode %{
12921     __ subw(as_Register($dst$$reg),
12922               as_Register($src1$$reg),
12923               as_Register($src2$$reg),
12924               Assembler::LSR,
12925               $src3$$constant & 0x1f);
12926   %}
12927 
12928   ins_pipe(ialu_reg_reg_shift);
12929 %}
12930 
12931 // This pattern is automatically generated from aarch64_ad.m4
12932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12933 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12934                          iRegL src1, iRegL src2,
12935                          immI src3) %{
12936   match(Set dst (SubL src1 (URShiftL src2 src3)));
12937 
12938   ins_cost(1.9 * INSN_COST);
12939   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12940 
12941   ins_encode %{
12942     __ sub(as_Register($dst$$reg),
12943               as_Register($src1$$reg),
12944               as_Register($src2$$reg),
12945               Assembler::LSR,
12946               $src3$$constant & 0x3f);
12947   %}
12948 
12949   ins_pipe(ialu_reg_reg_shift);
12950 %}
12951 
12952 // This pattern is automatically generated from aarch64_ad.m4
12953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12954 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12955                          iRegIorL2I src1, iRegIorL2I src2,
12956                          immI src3) %{
12957   match(Set dst (SubI src1 (RShiftI src2 src3)));
12958 
12959   ins_cost(1.9 * INSN_COST);
12960   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12961 
12962   ins_encode %{
12963     __ subw(as_Register($dst$$reg),
12964               as_Register($src1$$reg),
12965               as_Register($src2$$reg),
12966               Assembler::ASR,
12967               $src3$$constant & 0x1f);
12968   %}
12969 
12970   ins_pipe(ialu_reg_reg_shift);
12971 %}
12972 
12973 // This pattern is automatically generated from aarch64_ad.m4
12974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12975 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12976                          iRegL src1, iRegL src2,
12977                          immI src3) %{
12978   match(Set dst (SubL src1 (RShiftL src2 src3)));
12979 
12980   ins_cost(1.9 * INSN_COST);
12981   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12982 
12983   ins_encode %{
12984     __ sub(as_Register($dst$$reg),
12985               as_Register($src1$$reg),
12986               as_Register($src2$$reg),
12987               Assembler::ASR,
12988               $src3$$constant & 0x3f);
12989   %}
12990 
12991   ins_pipe(ialu_reg_reg_shift);
12992 %}
12993 
12994 // This pattern is automatically generated from aarch64_ad.m4
12995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12996 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12997                          iRegIorL2I src1, iRegIorL2I src2,
12998                          immI src3) %{
12999   match(Set dst (SubI src1 (LShiftI src2 src3)));
13000 
13001   ins_cost(1.9 * INSN_COST);
13002   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
13003 
13004   ins_encode %{
13005     __ subw(as_Register($dst$$reg),
13006               as_Register($src1$$reg),
13007               as_Register($src2$$reg),
13008               Assembler::LSL,
13009               $src3$$constant & 0x1f);
13010   %}
13011 
13012   ins_pipe(ialu_reg_reg_shift);
13013 %}
13014 
13015 // This pattern is automatically generated from aarch64_ad.m4
13016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13017 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
13018                          iRegL src1, iRegL src2,
13019                          immI src3) %{
13020   match(Set dst (SubL src1 (LShiftL src2 src3)));
13021 
13022   ins_cost(1.9 * INSN_COST);
13023   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
13024 
13025   ins_encode %{
13026     __ sub(as_Register($dst$$reg),
13027               as_Register($src1$$reg),
13028               as_Register($src2$$reg),
13029               Assembler::LSL,
13030               $src3$$constant & 0x3f);
13031   %}
13032 
13033   ins_pipe(ialu_reg_reg_shift);
13034 %}
13035 
13036 // This pattern is automatically generated from aarch64_ad.m4
13037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13038 
13039 // Shift Left followed by Shift Right.
13040 // This idiom is used by the compiler for the i2b bytecode etc.
13041 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13042 %{
13043   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
13044   ins_cost(INSN_COST * 2);
13045   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13046   ins_encode %{
13047     int lshift = $lshift_count$$constant & 63;
13048     int rshift = $rshift_count$$constant & 63;
13049     int s = 63 - lshift;
13050     int r = (rshift - lshift) & 63;
13051     __ sbfm(as_Register($dst$$reg),
13052             as_Register($src$$reg),
13053             r, s);
13054   %}
13055 
13056   ins_pipe(ialu_reg_shift);
13057 %}
13058 
13059 // This pattern is automatically generated from aarch64_ad.m4
13060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13061 
13062 // Shift Left followed by Shift Right.
13063 // This idiom is used by the compiler for the i2b bytecode etc.
13064 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13065 %{
13066   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
13067   ins_cost(INSN_COST * 2);
13068   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13069   ins_encode %{
13070     int lshift = $lshift_count$$constant & 31;
13071     int rshift = $rshift_count$$constant & 31;
13072     int s = 31 - lshift;
13073     int r = (rshift - lshift) & 31;
13074     __ sbfmw(as_Register($dst$$reg),
13075             as_Register($src$$reg),
13076             r, s);
13077   %}
13078 
13079   ins_pipe(ialu_reg_shift);
13080 %}
13081 
13082 // This pattern is automatically generated from aarch64_ad.m4
13083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13084 
13085 // Shift Left followed by Shift Right.
13086 // This idiom is used by the compiler for the i2b bytecode etc.
13087 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13088 %{
13089   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
13090   ins_cost(INSN_COST * 2);
13091   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13092   ins_encode %{
13093     int lshift = $lshift_count$$constant & 63;
13094     int rshift = $rshift_count$$constant & 63;
13095     int s = 63 - lshift;
13096     int r = (rshift - lshift) & 63;
13097     __ ubfm(as_Register($dst$$reg),
13098             as_Register($src$$reg),
13099             r, s);
13100   %}
13101 
13102   ins_pipe(ialu_reg_shift);
13103 %}
13104 
13105 // This pattern is automatically generated from aarch64_ad.m4
13106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13107 
13108 // Shift Left followed by Shift Right.
13109 // This idiom is used by the compiler for the i2b bytecode etc.
13110 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13111 %{
13112   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13113   ins_cost(INSN_COST * 2);
13114   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13115   ins_encode %{
13116     int lshift = $lshift_count$$constant & 31;
13117     int rshift = $rshift_count$$constant & 31;
13118     int s = 31 - lshift;
13119     int r = (rshift - lshift) & 31;
13120     __ ubfmw(as_Register($dst$$reg),
13121             as_Register($src$$reg),
13122             r, s);
13123   %}
13124 
13125   ins_pipe(ialu_reg_shift);
13126 %}
13127 
13128 // Bitfield extract with shift & mask
13129 
13130 // This pattern is automatically generated from aarch64_ad.m4
13131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13132 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13133 %{
13134   match(Set dst (AndI (URShiftI src rshift) mask));
13135   // Make sure we are not going to exceed what ubfxw can do.
13136   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13137 
13138   ins_cost(INSN_COST);
13139   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13140   ins_encode %{
13141     int rshift = $rshift$$constant & 31;
13142     intptr_t mask = $mask$$constant;
13143     int width = exact_log2(mask+1);
13144     __ ubfxw(as_Register($dst$$reg),
13145             as_Register($src$$reg), rshift, width);
13146   %}
13147   ins_pipe(ialu_reg_shift);
13148 %}
13149 
13150 // This pattern is automatically generated from aarch64_ad.m4
13151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13152 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13153 %{
13154   match(Set dst (AndL (URShiftL src rshift) mask));
13155   // Make sure we are not going to exceed what ubfx can do.
13156   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13157 
13158   ins_cost(INSN_COST);
13159   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13160   ins_encode %{
13161     int rshift = $rshift$$constant & 63;
13162     intptr_t mask = $mask$$constant;
13163     int width = exact_log2_long(mask+1);
13164     __ ubfx(as_Register($dst$$reg),
13165             as_Register($src$$reg), rshift, width);
13166   %}
13167   ins_pipe(ialu_reg_shift);
13168 %}
13169 
13170 
13171 // This pattern is automatically generated from aarch64_ad.m4
13172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13173 
13174 // We can use ubfx when extending an And with a mask when we know mask
13175 // is positive.  We know that because immI_bitmask guarantees it.
13176 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13177 %{
13178   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13179   // Make sure we are not going to exceed what ubfxw can do.
13180   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13181 
13182   ins_cost(INSN_COST * 2);
13183   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13184   ins_encode %{
13185     int rshift = $rshift$$constant & 31;
13186     intptr_t mask = $mask$$constant;
13187     int width = exact_log2(mask+1);
13188     __ ubfx(as_Register($dst$$reg),
13189             as_Register($src$$reg), rshift, width);
13190   %}
13191   ins_pipe(ialu_reg_shift);
13192 %}
13193 
13194 
13195 // This pattern is automatically generated from aarch64_ad.m4
13196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13197 
13198 // We can use ubfiz when masking by a positive number and then left shifting the result.
13199 // We know that the mask is positive because immI_bitmask guarantees it.
13200 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13201 %{
13202   match(Set dst (LShiftI (AndI src mask) lshift));
13203   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13204 
13205   ins_cost(INSN_COST);
13206   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13207   ins_encode %{
13208     int lshift = $lshift$$constant & 31;
13209     intptr_t mask = $mask$$constant;
13210     int width = exact_log2(mask+1);
13211     __ ubfizw(as_Register($dst$$reg),
13212           as_Register($src$$reg), lshift, width);
13213   %}
13214   ins_pipe(ialu_reg_shift);
13215 %}
13216 
13217 // This pattern is automatically generated from aarch64_ad.m4
13218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13219 
13220 // We can use ubfiz when masking by a positive number and then left shifting the result.
13221 // We know that the mask is positive because immL_bitmask guarantees it.
13222 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13223 %{
13224   match(Set dst (LShiftL (AndL src mask) lshift));
13225   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13226 
13227   ins_cost(INSN_COST);
13228   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13229   ins_encode %{
13230     int lshift = $lshift$$constant & 63;
13231     intptr_t mask = $mask$$constant;
13232     int width = exact_log2_long(mask+1);
13233     __ ubfiz(as_Register($dst$$reg),
13234           as_Register($src$$reg), lshift, width);
13235   %}
13236   ins_pipe(ialu_reg_shift);
13237 %}
13238 
13239 // This pattern is automatically generated from aarch64_ad.m4
13240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13241 
13242 // We can use ubfiz when masking by a positive number and then left shifting the result.
13243 // We know that the mask is positive because immI_bitmask guarantees it.
13244 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13245 %{
13246   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13247   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13248 
13249   ins_cost(INSN_COST);
13250   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13251   ins_encode %{
13252     int lshift = $lshift$$constant & 31;
13253     intptr_t mask = $mask$$constant;
13254     int width = exact_log2(mask+1);
13255     __ ubfizw(as_Register($dst$$reg),
13256           as_Register($src$$reg), lshift, width);
13257   %}
13258   ins_pipe(ialu_reg_shift);
13259 %}
13260 
13261 // This pattern is automatically generated from aarch64_ad.m4
13262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13263 
13264 // We can use ubfiz when masking by a positive number and then left shifting the result.
13265 // We know that the mask is positive because immL_bitmask guarantees it.
13266 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13267 %{
13268   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13269   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13270 
13271   ins_cost(INSN_COST);
13272   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13273   ins_encode %{
13274     int lshift = $lshift$$constant & 63;
13275     intptr_t mask = $mask$$constant;
13276     int width = exact_log2_long(mask+1);
13277     __ ubfiz(as_Register($dst$$reg),
13278           as_Register($src$$reg), lshift, width);
13279   %}
13280   ins_pipe(ialu_reg_shift);
13281 %}
13282 
13283 
13284 // This pattern is automatically generated from aarch64_ad.m4
13285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13286 
13287 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13288 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13289 %{
13290   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13291   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13292 
13293   ins_cost(INSN_COST);
13294   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13295   ins_encode %{
13296     int lshift = $lshift$$constant & 63;
13297     intptr_t mask = $mask$$constant;
13298     int width = exact_log2(mask+1);
13299     __ ubfiz(as_Register($dst$$reg),
13300              as_Register($src$$reg), lshift, width);
13301   %}
13302   ins_pipe(ialu_reg_shift);
13303 %}
13304 
13305 // This pattern is automatically generated from aarch64_ad.m4
13306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13307 
13308 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13309 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13310 %{
13311   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13312   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13313 
13314   ins_cost(INSN_COST);
13315   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13316   ins_encode %{
13317     int lshift = $lshift$$constant & 31;
13318     intptr_t mask = $mask$$constant;
13319     int width = exact_log2(mask+1);
13320     __ ubfiz(as_Register($dst$$reg),
13321              as_Register($src$$reg), lshift, width);
13322   %}
13323   ins_pipe(ialu_reg_shift);
13324 %}
13325 
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 
13329 // Can skip int2long conversions after AND with small bitmask
13330 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13331 %{
13332   match(Set dst (ConvI2L (AndI src msk)));
13333   ins_cost(INSN_COST);
13334   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13335   ins_encode %{
13336     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13337   %}
13338   ins_pipe(ialu_reg_shift);
13339 %}
13340 
13341 
13342 // Rotations
13343 // This pattern is automatically generated from aarch64_ad.m4
13344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13345 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13346 %{
13347   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13348   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13349 
13350   ins_cost(INSN_COST);
13351   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13352 
13353   ins_encode %{
13354     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13355             $rshift$$constant & 63);
13356   %}
13357   ins_pipe(ialu_reg_reg_extr);
13358 %}
13359 
13360 
13361 // This pattern is automatically generated from aarch64_ad.m4
13362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13363 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13364 %{
13365   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13366   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13367 
13368   ins_cost(INSN_COST);
13369   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13370 
13371   ins_encode %{
13372     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13373             $rshift$$constant & 31);
13374   %}
13375   ins_pipe(ialu_reg_reg_extr);
13376 %}
13377 
13378 
13379 // This pattern is automatically generated from aarch64_ad.m4
13380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13381 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13382 %{
13383   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13384   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13385 
13386   ins_cost(INSN_COST);
13387   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13388 
13389   ins_encode %{
13390     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13391             $rshift$$constant & 63);
13392   %}
13393   ins_pipe(ialu_reg_reg_extr);
13394 %}
13395 
13396 
13397 // This pattern is automatically generated from aarch64_ad.m4
13398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13399 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13400 %{
13401   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13402   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13403 
13404   ins_cost(INSN_COST);
13405   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13406 
13407   ins_encode %{
13408     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13409             $rshift$$constant & 31);
13410   %}
13411   ins_pipe(ialu_reg_reg_extr);
13412 %}
13413 
13414 
13415 // This pattern is automatically generated from aarch64_ad.m4
13416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13417 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13418 %{
13419   match(Set dst (RotateRight src shift));
13420 
13421   ins_cost(INSN_COST);
13422   format %{ "ror    $dst, $src, $shift" %}
13423 
13424   ins_encode %{
13425      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13426                $shift$$constant & 0x1f);
13427   %}
13428   ins_pipe(ialu_reg_reg_vshift);
13429 %}
13430 
13431 // This pattern is automatically generated from aarch64_ad.m4
13432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13433 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13434 %{
13435   match(Set dst (RotateRight src shift));
13436 
13437   ins_cost(INSN_COST);
13438   format %{ "ror    $dst, $src, $shift" %}
13439 
13440   ins_encode %{
13441      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13442                $shift$$constant & 0x3f);
13443   %}
13444   ins_pipe(ialu_reg_reg_vshift);
13445 %}
13446 
13447 // This pattern is automatically generated from aarch64_ad.m4
13448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13449 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13450 %{
13451   match(Set dst (RotateRight src shift));
13452 
13453   ins_cost(INSN_COST);
13454   format %{ "ror    $dst, $src, $shift" %}
13455 
13456   ins_encode %{
13457      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13458   %}
13459   ins_pipe(ialu_reg_reg_vshift);
13460 %}
13461 
13462 // This pattern is automatically generated from aarch64_ad.m4
13463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13464 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13465 %{
13466   match(Set dst (RotateRight src shift));
13467 
13468   ins_cost(INSN_COST);
13469   format %{ "ror    $dst, $src, $shift" %}
13470 
13471   ins_encode %{
13472      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13473   %}
13474   ins_pipe(ialu_reg_reg_vshift);
13475 %}
13476 
13477 // This pattern is automatically generated from aarch64_ad.m4
13478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13479 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13480 %{
13481   match(Set dst (RotateLeft src shift));
13482 
13483   ins_cost(INSN_COST);
13484   format %{ "rol    $dst, $src, $shift" %}
13485 
13486   ins_encode %{
13487      __ subw(rscratch1, zr, as_Register($shift$$reg));
13488      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13489   %}
13490   ins_pipe(ialu_reg_reg_vshift);
13491 %}
13492 
13493 // This pattern is automatically generated from aarch64_ad.m4
13494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13495 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13496 %{
13497   match(Set dst (RotateLeft src shift));
13498 
13499   ins_cost(INSN_COST);
13500   format %{ "rol    $dst, $src, $shift" %}
13501 
13502   ins_encode %{
13503      __ subw(rscratch1, zr, as_Register($shift$$reg));
13504      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13505   %}
13506   ins_pipe(ialu_reg_reg_vshift);
13507 %}
13508 
13509 
13510 // Add/subtract (extended)
13511 
13512 // This pattern is automatically generated from aarch64_ad.m4
13513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13514 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13515 %{
13516   match(Set dst (AddL src1 (ConvI2L src2)));
13517   ins_cost(INSN_COST);
13518   format %{ "add  $dst, $src1, $src2, sxtw" %}
13519 
13520    ins_encode %{
13521      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13522             as_Register($src2$$reg), ext::sxtw);
13523    %}
13524   ins_pipe(ialu_reg_reg);
13525 %}
13526 
13527 // This pattern is automatically generated from aarch64_ad.m4
13528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13529 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13530 %{
13531   match(Set dst (SubL src1 (ConvI2L src2)));
13532   ins_cost(INSN_COST);
13533   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13534 
13535    ins_encode %{
13536      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13537             as_Register($src2$$reg), ext::sxtw);
13538    %}
13539   ins_pipe(ialu_reg_reg);
13540 %}
13541 
13542 // This pattern is automatically generated from aarch64_ad.m4
13543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13544 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13545 %{
13546   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13547   ins_cost(INSN_COST);
13548   format %{ "add  $dst, $src1, $src2, sxth" %}
13549 
13550    ins_encode %{
13551      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13552             as_Register($src2$$reg), ext::sxth);
13553    %}
13554   ins_pipe(ialu_reg_reg);
13555 %}
13556 
13557 // This pattern is automatically generated from aarch64_ad.m4
13558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13559 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13560 %{
13561   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13562   ins_cost(INSN_COST);
13563   format %{ "add  $dst, $src1, $src2, sxtb" %}
13564 
13565    ins_encode %{
13566      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13567             as_Register($src2$$reg), ext::sxtb);
13568    %}
13569   ins_pipe(ialu_reg_reg);
13570 %}
13571 
13572 // This pattern is automatically generated from aarch64_ad.m4
13573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13574 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13575 %{
13576   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13577   ins_cost(INSN_COST);
13578   format %{ "add  $dst, $src1, $src2, uxtb" %}
13579 
13580    ins_encode %{
13581      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13582             as_Register($src2$$reg), ext::uxtb);
13583    %}
13584   ins_pipe(ialu_reg_reg);
13585 %}
13586 
13587 // This pattern is automatically generated from aarch64_ad.m4
13588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13589 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13590 %{
13591   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13592   ins_cost(INSN_COST);
13593   format %{ "add  $dst, $src1, $src2, sxth" %}
13594 
13595    ins_encode %{
13596      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13597             as_Register($src2$$reg), ext::sxth);
13598    %}
13599   ins_pipe(ialu_reg_reg);
13600 %}
13601 
13602 // This pattern is automatically generated from aarch64_ad.m4
13603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13604 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13605 %{
13606   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13607   ins_cost(INSN_COST);
13608   format %{ "add  $dst, $src1, $src2, sxtw" %}
13609 
13610    ins_encode %{
13611      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13612             as_Register($src2$$reg), ext::sxtw);
13613    %}
13614   ins_pipe(ialu_reg_reg);
13615 %}
13616 
13617 // This pattern is automatically generated from aarch64_ad.m4
13618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13619 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13620 %{
13621   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13622   ins_cost(INSN_COST);
13623   format %{ "add  $dst, $src1, $src2, sxtb" %}
13624 
13625    ins_encode %{
13626      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13627             as_Register($src2$$reg), ext::sxtb);
13628    %}
13629   ins_pipe(ialu_reg_reg);
13630 %}
13631 
13632 // This pattern is automatically generated from aarch64_ad.m4
13633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13634 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13635 %{
13636   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13637   ins_cost(INSN_COST);
13638   format %{ "add  $dst, $src1, $src2, uxtb" %}
13639 
13640    ins_encode %{
13641      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13642             as_Register($src2$$reg), ext::uxtb);
13643    %}
13644   ins_pipe(ialu_reg_reg);
13645 %}
13646 
13647 // This pattern is automatically generated from aarch64_ad.m4
13648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13649 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13650 %{
13651   match(Set dst (AddI src1 (AndI src2 mask)));
13652   ins_cost(INSN_COST);
13653   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13654 
13655    ins_encode %{
13656      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13657             as_Register($src2$$reg), ext::uxtb);
13658    %}
13659   ins_pipe(ialu_reg_reg);
13660 %}
13661 
13662 // This pattern is automatically generated from aarch64_ad.m4
13663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13664 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13665 %{
13666   match(Set dst (AddI src1 (AndI src2 mask)));
13667   ins_cost(INSN_COST);
13668   format %{ "addw  $dst, $src1, $src2, uxth" %}
13669 
13670    ins_encode %{
13671      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13672             as_Register($src2$$reg), ext::uxth);
13673    %}
13674   ins_pipe(ialu_reg_reg);
13675 %}
13676 
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13680 %{
13681   match(Set dst (AddL src1 (AndL src2 mask)));
13682   ins_cost(INSN_COST);
13683   format %{ "add  $dst, $src1, $src2, uxtb" %}
13684 
13685    ins_encode %{
13686      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13687             as_Register($src2$$reg), ext::uxtb);
13688    %}
13689   ins_pipe(ialu_reg_reg);
13690 %}
13691 
13692 // This pattern is automatically generated from aarch64_ad.m4
13693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13694 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13695 %{
13696   match(Set dst (AddL src1 (AndL src2 mask)));
13697   ins_cost(INSN_COST);
13698   format %{ "add  $dst, $src1, $src2, uxth" %}
13699 
13700    ins_encode %{
13701      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13702             as_Register($src2$$reg), ext::uxth);
13703    %}
13704   ins_pipe(ialu_reg_reg);
13705 %}
13706 
13707 // This pattern is automatically generated from aarch64_ad.m4
13708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13709 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13710 %{
13711   match(Set dst (AddL src1 (AndL src2 mask)));
13712   ins_cost(INSN_COST);
13713   format %{ "add  $dst, $src1, $src2, uxtw" %}
13714 
13715    ins_encode %{
13716      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13717             as_Register($src2$$reg), ext::uxtw);
13718    %}
13719   ins_pipe(ialu_reg_reg);
13720 %}
13721 
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13725 %{
13726   match(Set dst (SubI src1 (AndI src2 mask)));
13727   ins_cost(INSN_COST);
13728   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13729 
13730    ins_encode %{
13731      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13732             as_Register($src2$$reg), ext::uxtb);
13733    %}
13734   ins_pipe(ialu_reg_reg);
13735 %}
13736 
13737 // This pattern is automatically generated from aarch64_ad.m4
13738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13739 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13740 %{
13741   match(Set dst (SubI src1 (AndI src2 mask)));
13742   ins_cost(INSN_COST);
13743   format %{ "subw  $dst, $src1, $src2, uxth" %}
13744 
13745    ins_encode %{
13746      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13747             as_Register($src2$$reg), ext::uxth);
13748    %}
13749   ins_pipe(ialu_reg_reg);
13750 %}
13751 
13752 // This pattern is automatically generated from aarch64_ad.m4
13753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13754 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13755 %{
13756   match(Set dst (SubL src1 (AndL src2 mask)));
13757   ins_cost(INSN_COST);
13758   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13759 
13760    ins_encode %{
13761      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13762             as_Register($src2$$reg), ext::uxtb);
13763    %}
13764   ins_pipe(ialu_reg_reg);
13765 %}
13766 
13767 // This pattern is automatically generated from aarch64_ad.m4
13768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13769 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13770 %{
13771   match(Set dst (SubL src1 (AndL src2 mask)));
13772   ins_cost(INSN_COST);
13773   format %{ "sub  $dst, $src1, $src2, uxth" %}
13774 
13775    ins_encode %{
13776      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13777             as_Register($src2$$reg), ext::uxth);
13778    %}
13779   ins_pipe(ialu_reg_reg);
13780 %}
13781 
13782 // This pattern is automatically generated from aarch64_ad.m4
13783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13784 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13785 %{
13786   match(Set dst (SubL src1 (AndL src2 mask)));
13787   ins_cost(INSN_COST);
13788   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13789 
13790    ins_encode %{
13791      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13792             as_Register($src2$$reg), ext::uxtw);
13793    %}
13794   ins_pipe(ialu_reg_reg);
13795 %}
13796 
13797 
13798 // This pattern is automatically generated from aarch64_ad.m4
13799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13800 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13801 %{
13802   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13803   ins_cost(1.9 * INSN_COST);
13804   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13805 
13806    ins_encode %{
13807      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13808             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13809    %}
13810   ins_pipe(ialu_reg_reg_shift);
13811 %}
13812 
13813 // This pattern is automatically generated from aarch64_ad.m4
13814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13815 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13816 %{
13817   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13818   ins_cost(1.9 * INSN_COST);
13819   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13820 
13821    ins_encode %{
13822      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13823             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13824    %}
13825   ins_pipe(ialu_reg_reg_shift);
13826 %}
13827 
13828 // This pattern is automatically generated from aarch64_ad.m4
13829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13830 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13831 %{
13832   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13833   ins_cost(1.9 * INSN_COST);
13834   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13835 
13836    ins_encode %{
13837      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13838             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13839    %}
13840   ins_pipe(ialu_reg_reg_shift);
13841 %}
13842 
13843 // This pattern is automatically generated from aarch64_ad.m4
13844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13845 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13846 %{
13847   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13848   ins_cost(1.9 * INSN_COST);
13849   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13850 
13851    ins_encode %{
13852      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13853             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13854    %}
13855   ins_pipe(ialu_reg_reg_shift);
13856 %}
13857 
13858 // This pattern is automatically generated from aarch64_ad.m4
13859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13860 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13861 %{
13862   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13863   ins_cost(1.9 * INSN_COST);
13864   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13865 
13866    ins_encode %{
13867      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13868             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13869    %}
13870   ins_pipe(ialu_reg_reg_shift);
13871 %}
13872 
13873 // This pattern is automatically generated from aarch64_ad.m4
13874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13875 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13876 %{
13877   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13878   ins_cost(1.9 * INSN_COST);
13879   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13880 
13881    ins_encode %{
13882      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13883             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13884    %}
13885   ins_pipe(ialu_reg_reg_shift);
13886 %}
13887 
13888 // This pattern is automatically generated from aarch64_ad.m4
13889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13890 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13891 %{
13892   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13893   ins_cost(1.9 * INSN_COST);
13894   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13895 
13896    ins_encode %{
13897      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13898             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13899    %}
13900   ins_pipe(ialu_reg_reg_shift);
13901 %}
13902 
13903 // This pattern is automatically generated from aarch64_ad.m4
13904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13905 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13906 %{
13907   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13908   ins_cost(1.9 * INSN_COST);
13909   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13910 
13911    ins_encode %{
13912      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13913             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13914    %}
13915   ins_pipe(ialu_reg_reg_shift);
13916 %}
13917 
13918 // This pattern is automatically generated from aarch64_ad.m4
13919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13920 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13921 %{
13922   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13923   ins_cost(1.9 * INSN_COST);
13924   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13925 
13926    ins_encode %{
13927      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13928             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13929    %}
13930   ins_pipe(ialu_reg_reg_shift);
13931 %}
13932 
13933 // This pattern is automatically generated from aarch64_ad.m4
13934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13935 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13936 %{
13937   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13938   ins_cost(1.9 * INSN_COST);
13939   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13940 
13941    ins_encode %{
13942      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13943             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13944    %}
13945   ins_pipe(ialu_reg_reg_shift);
13946 %}
13947 
13948 // This pattern is automatically generated from aarch64_ad.m4
13949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13950 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13951 %{
13952   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13953   ins_cost(1.9 * INSN_COST);
13954   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13955 
13956    ins_encode %{
13957      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13958             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13959    %}
13960   ins_pipe(ialu_reg_reg_shift);
13961 %}
13962 
13963 // This pattern is automatically generated from aarch64_ad.m4
13964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13965 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13966 %{
13967   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13968   ins_cost(1.9 * INSN_COST);
13969   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13970 
13971    ins_encode %{
13972      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13973             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13974    %}
13975   ins_pipe(ialu_reg_reg_shift);
13976 %}
13977 
13978 // This pattern is automatically generated from aarch64_ad.m4
13979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13980 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13981 %{
13982   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13983   ins_cost(1.9 * INSN_COST);
13984   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13985 
13986    ins_encode %{
13987      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13988             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13989    %}
13990   ins_pipe(ialu_reg_reg_shift);
13991 %}
13992 
13993 // This pattern is automatically generated from aarch64_ad.m4
13994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13995 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13996 %{
13997   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13998   ins_cost(1.9 * INSN_COST);
13999   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
14000 
14001    ins_encode %{
14002      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14003             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14004    %}
14005   ins_pipe(ialu_reg_reg_shift);
14006 %}
14007 
14008 // This pattern is automatically generated from aarch64_ad.m4
14009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14010 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14011 %{
14012   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14013   ins_cost(1.9 * INSN_COST);
14014   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
14015 
14016    ins_encode %{
14017      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14018             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14019    %}
14020   ins_pipe(ialu_reg_reg_shift);
14021 %}
14022 
14023 // This pattern is automatically generated from aarch64_ad.m4
14024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14025 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14026 %{
14027   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14028   ins_cost(1.9 * INSN_COST);
14029   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
14030 
14031    ins_encode %{
14032      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14033             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14034    %}
14035   ins_pipe(ialu_reg_reg_shift);
14036 %}
14037 
14038 // This pattern is automatically generated from aarch64_ad.m4
14039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14040 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14041 %{
14042   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14043   ins_cost(1.9 * INSN_COST);
14044   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
14045 
14046    ins_encode %{
14047      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14048             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14049    %}
14050   ins_pipe(ialu_reg_reg_shift);
14051 %}
14052 
14053 // This pattern is automatically generated from aarch64_ad.m4
14054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14055 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14056 %{
14057   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14058   ins_cost(1.9 * INSN_COST);
14059   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
14060 
14061    ins_encode %{
14062      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14063             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14064    %}
14065   ins_pipe(ialu_reg_reg_shift);
14066 %}
14067 
14068 // This pattern is automatically generated from aarch64_ad.m4
14069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14070 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14071 %{
14072   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14073   ins_cost(1.9 * INSN_COST);
14074   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
14075 
14076    ins_encode %{
14077      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14078             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14079    %}
14080   ins_pipe(ialu_reg_reg_shift);
14081 %}
14082 
14083 // This pattern is automatically generated from aarch64_ad.m4
14084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14085 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14086 %{
14087   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14088   ins_cost(1.9 * INSN_COST);
14089   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
14090 
14091    ins_encode %{
14092      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14093             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14094    %}
14095   ins_pipe(ialu_reg_reg_shift);
14096 %}
14097 
14098 // This pattern is automatically generated from aarch64_ad.m4
14099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14100 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14101 %{
14102   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14103   ins_cost(1.9 * INSN_COST);
14104   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
14105 
14106    ins_encode %{
14107      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14108             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14109    %}
14110   ins_pipe(ialu_reg_reg_shift);
14111 %}
14112 
14113 // This pattern is automatically generated from aarch64_ad.m4
14114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14115 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14116 %{
14117   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14118   ins_cost(1.9 * INSN_COST);
14119   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
14120 
14121    ins_encode %{
14122      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14123             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14124    %}
14125   ins_pipe(ialu_reg_reg_shift);
14126 %}
14127 
14128 
14129 
14130 // END This section of the file is automatically generated. Do not edit --------------
14131 
14132 
14133 // ============================================================================
14134 // Floating Point Arithmetic Instructions
14135 
14136 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14137   match(Set dst (AddF src1 src2));
14138 
14139   ins_cost(INSN_COST * 5);
14140   format %{ "fadds   $dst, $src1, $src2" %}
14141 
14142   ins_encode %{
14143     __ fadds(as_FloatRegister($dst$$reg),
14144              as_FloatRegister($src1$$reg),
14145              as_FloatRegister($src2$$reg));
14146   %}
14147 
14148   ins_pipe(fp_dop_reg_reg_s);
14149 %}
14150 
14151 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14152   match(Set dst (AddD src1 src2));
14153 
14154   ins_cost(INSN_COST * 5);
14155   format %{ "faddd   $dst, $src1, $src2" %}
14156 
14157   ins_encode %{
14158     __ faddd(as_FloatRegister($dst$$reg),
14159              as_FloatRegister($src1$$reg),
14160              as_FloatRegister($src2$$reg));
14161   %}
14162 
14163   ins_pipe(fp_dop_reg_reg_d);
14164 %}
14165 
14166 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14167   match(Set dst (SubF src1 src2));
14168 
14169   ins_cost(INSN_COST * 5);
14170   format %{ "fsubs   $dst, $src1, $src2" %}
14171 
14172   ins_encode %{
14173     __ fsubs(as_FloatRegister($dst$$reg),
14174              as_FloatRegister($src1$$reg),
14175              as_FloatRegister($src2$$reg));
14176   %}
14177 
14178   ins_pipe(fp_dop_reg_reg_s);
14179 %}
14180 
14181 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14182   match(Set dst (SubD src1 src2));
14183 
14184   ins_cost(INSN_COST * 5);
14185   format %{ "fsubd   $dst, $src1, $src2" %}
14186 
14187   ins_encode %{
14188     __ fsubd(as_FloatRegister($dst$$reg),
14189              as_FloatRegister($src1$$reg),
14190              as_FloatRegister($src2$$reg));
14191   %}
14192 
14193   ins_pipe(fp_dop_reg_reg_d);
14194 %}
14195 
14196 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14197   match(Set dst (MulF src1 src2));
14198 
14199   ins_cost(INSN_COST * 6);
14200   format %{ "fmuls   $dst, $src1, $src2" %}
14201 
14202   ins_encode %{
14203     __ fmuls(as_FloatRegister($dst$$reg),
14204              as_FloatRegister($src1$$reg),
14205              as_FloatRegister($src2$$reg));
14206   %}
14207 
14208   ins_pipe(fp_dop_reg_reg_s);
14209 %}
14210 
14211 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14212   match(Set dst (MulD src1 src2));
14213 
14214   ins_cost(INSN_COST * 6);
14215   format %{ "fmuld   $dst, $src1, $src2" %}
14216 
14217   ins_encode %{
14218     __ fmuld(as_FloatRegister($dst$$reg),
14219              as_FloatRegister($src1$$reg),
14220              as_FloatRegister($src2$$reg));
14221   %}
14222 
14223   ins_pipe(fp_dop_reg_reg_d);
14224 %}
14225 
14226 // src1 * src2 + src3
14227 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14228   predicate(UseFMA);
14229   match(Set dst (FmaF src3 (Binary src1 src2)));
14230 
14231   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14232 
14233   ins_encode %{
14234     __ fmadds(as_FloatRegister($dst$$reg),
14235              as_FloatRegister($src1$$reg),
14236              as_FloatRegister($src2$$reg),
14237              as_FloatRegister($src3$$reg));
14238   %}
14239 
14240   ins_pipe(pipe_class_default);
14241 %}
14242 
14243 // src1 * src2 + src3
14244 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14245   predicate(UseFMA);
14246   match(Set dst (FmaD src3 (Binary src1 src2)));
14247 
14248   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14249 
14250   ins_encode %{
14251     __ fmaddd(as_FloatRegister($dst$$reg),
14252              as_FloatRegister($src1$$reg),
14253              as_FloatRegister($src2$$reg),
14254              as_FloatRegister($src3$$reg));
14255   %}
14256 
14257   ins_pipe(pipe_class_default);
14258 %}
14259 
14260 // -src1 * src2 + src3
14261 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14262   predicate(UseFMA);
14263   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14264   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14265 
14266   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14267 
14268   ins_encode %{
14269     __ fmsubs(as_FloatRegister($dst$$reg),
14270               as_FloatRegister($src1$$reg),
14271               as_FloatRegister($src2$$reg),
14272               as_FloatRegister($src3$$reg));
14273   %}
14274 
14275   ins_pipe(pipe_class_default);
14276 %}
14277 
14278 // -src1 * src2 + src3
14279 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14280   predicate(UseFMA);
14281   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14282   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14283 
14284   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14285 
14286   ins_encode %{
14287     __ fmsubd(as_FloatRegister($dst$$reg),
14288               as_FloatRegister($src1$$reg),
14289               as_FloatRegister($src2$$reg),
14290               as_FloatRegister($src3$$reg));
14291   %}
14292 
14293   ins_pipe(pipe_class_default);
14294 %}
14295 
14296 // -src1 * src2 - src3
14297 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14298   predicate(UseFMA);
14299   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14300   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14301 
14302   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14303 
14304   ins_encode %{
14305     __ fnmadds(as_FloatRegister($dst$$reg),
14306                as_FloatRegister($src1$$reg),
14307                as_FloatRegister($src2$$reg),
14308                as_FloatRegister($src3$$reg));
14309   %}
14310 
14311   ins_pipe(pipe_class_default);
14312 %}
14313 
14314 // -src1 * src2 - src3
14315 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14316   predicate(UseFMA);
14317   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14318   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14319 
14320   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14321 
14322   ins_encode %{
14323     __ fnmaddd(as_FloatRegister($dst$$reg),
14324                as_FloatRegister($src1$$reg),
14325                as_FloatRegister($src2$$reg),
14326                as_FloatRegister($src3$$reg));
14327   %}
14328 
14329   ins_pipe(pipe_class_default);
14330 %}
14331 
14332 // src1 * src2 - src3
14333 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14334   predicate(UseFMA);
14335   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14336 
14337   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14338 
14339   ins_encode %{
14340     __ fnmsubs(as_FloatRegister($dst$$reg),
14341                as_FloatRegister($src1$$reg),
14342                as_FloatRegister($src2$$reg),
14343                as_FloatRegister($src3$$reg));
14344   %}
14345 
14346   ins_pipe(pipe_class_default);
14347 %}
14348 
14349 // src1 * src2 - src3
14350 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14351   predicate(UseFMA);
14352   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14353 
14354   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14355 
14356   ins_encode %{
14357   // n.b. insn name should be fnmsubd
14358     __ fnmsub(as_FloatRegister($dst$$reg),
14359               as_FloatRegister($src1$$reg),
14360               as_FloatRegister($src2$$reg),
14361               as_FloatRegister($src3$$reg));
14362   %}
14363 
14364   ins_pipe(pipe_class_default);
14365 %}
14366 
14367 
14368 // Math.max(FF)F
14369 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14370   match(Set dst (MaxF src1 src2));
14371 
14372   format %{ "fmaxs   $dst, $src1, $src2" %}
14373   ins_encode %{
14374     __ fmaxs(as_FloatRegister($dst$$reg),
14375              as_FloatRegister($src1$$reg),
14376              as_FloatRegister($src2$$reg));
14377   %}
14378 
14379   ins_pipe(fp_dop_reg_reg_s);
14380 %}
14381 
14382 // Math.min(FF)F
14383 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14384   match(Set dst (MinF src1 src2));
14385 
14386   format %{ "fmins   $dst, $src1, $src2" %}
14387   ins_encode %{
14388     __ fmins(as_FloatRegister($dst$$reg),
14389              as_FloatRegister($src1$$reg),
14390              as_FloatRegister($src2$$reg));
14391   %}
14392 
14393   ins_pipe(fp_dop_reg_reg_s);
14394 %}
14395 
14396 // Math.max(DD)D
14397 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14398   match(Set dst (MaxD src1 src2));
14399 
14400   format %{ "fmaxd   $dst, $src1, $src2" %}
14401   ins_encode %{
14402     __ fmaxd(as_FloatRegister($dst$$reg),
14403              as_FloatRegister($src1$$reg),
14404              as_FloatRegister($src2$$reg));
14405   %}
14406 
14407   ins_pipe(fp_dop_reg_reg_d);
14408 %}
14409 
14410 // Math.min(DD)D
14411 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14412   match(Set dst (MinD src1 src2));
14413 
14414   format %{ "fmind   $dst, $src1, $src2" %}
14415   ins_encode %{
14416     __ fmind(as_FloatRegister($dst$$reg),
14417              as_FloatRegister($src1$$reg),
14418              as_FloatRegister($src2$$reg));
14419   %}
14420 
14421   ins_pipe(fp_dop_reg_reg_d);
14422 %}
14423 
14424 
14425 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14426   match(Set dst (DivF src1  src2));
14427 
14428   ins_cost(INSN_COST * 18);
14429   format %{ "fdivs   $dst, $src1, $src2" %}
14430 
14431   ins_encode %{
14432     __ fdivs(as_FloatRegister($dst$$reg),
14433              as_FloatRegister($src1$$reg),
14434              as_FloatRegister($src2$$reg));
14435   %}
14436 
14437   ins_pipe(fp_div_s);
14438 %}
14439 
14440 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14441   match(Set dst (DivD src1  src2));
14442 
14443   ins_cost(INSN_COST * 32);
14444   format %{ "fdivd   $dst, $src1, $src2" %}
14445 
14446   ins_encode %{
14447     __ fdivd(as_FloatRegister($dst$$reg),
14448              as_FloatRegister($src1$$reg),
14449              as_FloatRegister($src2$$reg));
14450   %}
14451 
14452   ins_pipe(fp_div_d);
14453 %}
14454 
14455 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14456   match(Set dst (NegF src));
14457 
14458   ins_cost(INSN_COST * 3);
14459   format %{ "fneg   $dst, $src" %}
14460 
14461   ins_encode %{
14462     __ fnegs(as_FloatRegister($dst$$reg),
14463              as_FloatRegister($src$$reg));
14464   %}
14465 
14466   ins_pipe(fp_uop_s);
14467 %}
14468 
14469 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14470   match(Set dst (NegD src));
14471 
14472   ins_cost(INSN_COST * 3);
14473   format %{ "fnegd   $dst, $src" %}
14474 
14475   ins_encode %{
14476     __ fnegd(as_FloatRegister($dst$$reg),
14477              as_FloatRegister($src$$reg));
14478   %}
14479 
14480   ins_pipe(fp_uop_d);
14481 %}
14482 
14483 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14484 %{
14485   match(Set dst (AbsI src));
14486 
14487   effect(KILL cr);
14488   ins_cost(INSN_COST * 2);
14489   format %{ "cmpw  $src, zr\n\t"
14490             "cnegw $dst, $src, Assembler::LT\t# int abs"
14491   %}
14492 
14493   ins_encode %{
14494     __ cmpw(as_Register($src$$reg), zr);
14495     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14496   %}
14497   ins_pipe(pipe_class_default);
14498 %}
14499 
14500 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14501 %{
14502   match(Set dst (AbsL src));
14503 
14504   effect(KILL cr);
14505   ins_cost(INSN_COST * 2);
14506   format %{ "cmp  $src, zr\n\t"
14507             "cneg $dst, $src, Assembler::LT\t# long abs"
14508   %}
14509 
14510   ins_encode %{
14511     __ cmp(as_Register($src$$reg), zr);
14512     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14513   %}
14514   ins_pipe(pipe_class_default);
14515 %}
14516 
14517 instruct absF_reg(vRegF dst, vRegF src) %{
14518   match(Set dst (AbsF src));
14519 
14520   ins_cost(INSN_COST * 3);
14521   format %{ "fabss   $dst, $src" %}
14522   ins_encode %{
14523     __ fabss(as_FloatRegister($dst$$reg),
14524              as_FloatRegister($src$$reg));
14525   %}
14526 
14527   ins_pipe(fp_uop_s);
14528 %}
14529 
14530 instruct absD_reg(vRegD dst, vRegD src) %{
14531   match(Set dst (AbsD src));
14532 
14533   ins_cost(INSN_COST * 3);
14534   format %{ "fabsd   $dst, $src" %}
14535   ins_encode %{
14536     __ fabsd(as_FloatRegister($dst$$reg),
14537              as_FloatRegister($src$$reg));
14538   %}
14539 
14540   ins_pipe(fp_uop_d);
14541 %}
14542 
14543 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14544   match(Set dst (AbsF (SubF src1 src2)));
14545 
14546   ins_cost(INSN_COST * 3);
14547   format %{ "fabds   $dst, $src1, $src2" %}
14548   ins_encode %{
14549     __ fabds(as_FloatRegister($dst$$reg),
14550              as_FloatRegister($src1$$reg),
14551              as_FloatRegister($src2$$reg));
14552   %}
14553 
14554   ins_pipe(fp_uop_s);
14555 %}
14556 
14557 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14558   match(Set dst (AbsD (SubD src1 src2)));
14559 
14560   ins_cost(INSN_COST * 3);
14561   format %{ "fabdd   $dst, $src1, $src2" %}
14562   ins_encode %{
14563     __ fabdd(as_FloatRegister($dst$$reg),
14564              as_FloatRegister($src1$$reg),
14565              as_FloatRegister($src2$$reg));
14566   %}
14567 
14568   ins_pipe(fp_uop_d);
14569 %}
14570 
14571 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14572   match(Set dst (SqrtD src));
14573 
14574   ins_cost(INSN_COST * 50);
14575   format %{ "fsqrtd  $dst, $src" %}
14576   ins_encode %{
14577     __ fsqrtd(as_FloatRegister($dst$$reg),
14578              as_FloatRegister($src$$reg));
14579   %}
14580 
14581   ins_pipe(fp_div_s);
14582 %}
14583 
14584 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14585   match(Set dst (SqrtF src));
14586 
14587   ins_cost(INSN_COST * 50);
14588   format %{ "fsqrts  $dst, $src" %}
14589   ins_encode %{
14590     __ fsqrts(as_FloatRegister($dst$$reg),
14591              as_FloatRegister($src$$reg));
14592   %}
14593 
14594   ins_pipe(fp_div_d);
14595 %}
14596 
14597 // Math.rint, floor, ceil
14598 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14599   match(Set dst (RoundDoubleMode src rmode));
14600   format %{ "frint  $dst, $src, $rmode" %}
14601   ins_encode %{
14602     switch ($rmode$$constant) {
14603       case RoundDoubleModeNode::rmode_rint:
14604         __ frintnd(as_FloatRegister($dst$$reg),
14605                    as_FloatRegister($src$$reg));
14606         break;
14607       case RoundDoubleModeNode::rmode_floor:
14608         __ frintmd(as_FloatRegister($dst$$reg),
14609                    as_FloatRegister($src$$reg));
14610         break;
14611       case RoundDoubleModeNode::rmode_ceil:
14612         __ frintpd(as_FloatRegister($dst$$reg),
14613                    as_FloatRegister($src$$reg));
14614         break;
14615     }
14616   %}
14617   ins_pipe(fp_uop_d);
14618 %}
14619 
14620 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14621   match(Set dst (CopySignD src1 (Binary src2 zero)));
14622   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14623   format %{ "CopySignD  $dst $src1 $src2" %}
14624   ins_encode %{
14625     FloatRegister dst = as_FloatRegister($dst$$reg),
14626                   src1 = as_FloatRegister($src1$$reg),
14627                   src2 = as_FloatRegister($src2$$reg),
14628                   zero = as_FloatRegister($zero$$reg);
14629     __ fnegd(dst, zero);
14630     __ bsl(dst, __ T8B, src2, src1);
14631   %}
14632   ins_pipe(fp_uop_d);
14633 %}
14634 
14635 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14636   match(Set dst (CopySignF src1 src2));
14637   effect(TEMP_DEF dst, USE src1, USE src2);
14638   format %{ "CopySignF  $dst $src1 $src2" %}
14639   ins_encode %{
14640     FloatRegister dst = as_FloatRegister($dst$$reg),
14641                   src1 = as_FloatRegister($src1$$reg),
14642                   src2 = as_FloatRegister($src2$$reg);
14643     __ movi(dst, __ T2S, 0x80, 24);
14644     __ bsl(dst, __ T8B, src2, src1);
14645   %}
14646   ins_pipe(fp_uop_d);
14647 %}
14648 
14649 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14650   match(Set dst (SignumD src (Binary zero one)));
14651   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14652   format %{ "signumD  $dst, $src" %}
14653   ins_encode %{
14654     FloatRegister src = as_FloatRegister($src$$reg),
14655                   dst = as_FloatRegister($dst$$reg),
14656                   zero = as_FloatRegister($zero$$reg),
14657                   one = as_FloatRegister($one$$reg);
14658     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14659     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14660     // Bit selection instruction gets bit from "one" for each enabled bit in
14661     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14662     // NaN the whole "src" will be copied because "dst" is zero. For all other
14663     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14664     // from "src", and all other bits are copied from 1.0.
14665     __ bsl(dst, __ T8B, one, src);
14666   %}
14667   ins_pipe(fp_uop_d);
14668 %}
14669 
14670 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14671   match(Set dst (SignumF src (Binary zero one)));
14672   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14673   format %{ "signumF  $dst, $src" %}
14674   ins_encode %{
14675     FloatRegister src = as_FloatRegister($src$$reg),
14676                   dst = as_FloatRegister($dst$$reg),
14677                   zero = as_FloatRegister($zero$$reg),
14678                   one = as_FloatRegister($one$$reg);
14679     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14680     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14681     // Bit selection instruction gets bit from "one" for each enabled bit in
14682     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14683     // NaN the whole "src" will be copied because "dst" is zero. For all other
14684     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14685     // from "src", and all other bits are copied from 1.0.
14686     __ bsl(dst, __ T8B, one, src);
14687   %}
14688   ins_pipe(fp_uop_d);
14689 %}
14690 
14691 instruct onspinwait() %{
14692   match(OnSpinWait);
14693   ins_cost(INSN_COST);
14694 
14695   format %{ "onspinwait" %}
14696 
14697   ins_encode %{
14698     __ spin_wait();
14699   %}
14700   ins_pipe(pipe_class_empty);
14701 %}
14702 
14703 // ============================================================================
14704 // Logical Instructions
14705 
14706 // Integer Logical Instructions
14707 
14708 // And Instructions
14709 
14710 
14711 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14712   match(Set dst (AndI src1 src2));
14713 
14714   format %{ "andw  $dst, $src1, $src2\t# int" %}
14715 
14716   ins_cost(INSN_COST);
14717   ins_encode %{
14718     __ andw(as_Register($dst$$reg),
14719             as_Register($src1$$reg),
14720             as_Register($src2$$reg));
14721   %}
14722 
14723   ins_pipe(ialu_reg_reg);
14724 %}
14725 
14726 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14727   match(Set dst (AndI src1 src2));
14728 
14729   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14730 
14731   ins_cost(INSN_COST);
14732   ins_encode %{
14733     __ andw(as_Register($dst$$reg),
14734             as_Register($src1$$reg),
14735             (uint64_t)($src2$$constant));
14736   %}
14737 
14738   ins_pipe(ialu_reg_imm);
14739 %}
14740 
14741 // Or Instructions
14742 
14743 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14744   match(Set dst (OrI src1 src2));
14745 
14746   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14747 
14748   ins_cost(INSN_COST);
14749   ins_encode %{
14750     __ orrw(as_Register($dst$$reg),
14751             as_Register($src1$$reg),
14752             as_Register($src2$$reg));
14753   %}
14754 
14755   ins_pipe(ialu_reg_reg);
14756 %}
14757 
14758 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14759   match(Set dst (OrI src1 src2));
14760 
14761   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14762 
14763   ins_cost(INSN_COST);
14764   ins_encode %{
14765     __ orrw(as_Register($dst$$reg),
14766             as_Register($src1$$reg),
14767             (uint64_t)($src2$$constant));
14768   %}
14769 
14770   ins_pipe(ialu_reg_imm);
14771 %}
14772 
14773 // Xor Instructions
14774 
14775 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14776   match(Set dst (XorI src1 src2));
14777 
14778   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14779 
14780   ins_cost(INSN_COST);
14781   ins_encode %{
14782     __ eorw(as_Register($dst$$reg),
14783             as_Register($src1$$reg),
14784             as_Register($src2$$reg));
14785   %}
14786 
14787   ins_pipe(ialu_reg_reg);
14788 %}
14789 
14790 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14791   match(Set dst (XorI src1 src2));
14792 
14793   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14794 
14795   ins_cost(INSN_COST);
14796   ins_encode %{
14797     __ eorw(as_Register($dst$$reg),
14798             as_Register($src1$$reg),
14799             (uint64_t)($src2$$constant));
14800   %}
14801 
14802   ins_pipe(ialu_reg_imm);
14803 %}
14804 
14805 // Long Logical Instructions
14806 // TODO
14807 
14808 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14809   match(Set dst (AndL src1 src2));
14810 
14811   format %{ "and  $dst, $src1, $src2\t# int" %}
14812 
14813   ins_cost(INSN_COST);
14814   ins_encode %{
14815     __ andr(as_Register($dst$$reg),
14816             as_Register($src1$$reg),
14817             as_Register($src2$$reg));
14818   %}
14819 
14820   ins_pipe(ialu_reg_reg);
14821 %}
14822 
14823 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14824   match(Set dst (AndL src1 src2));
14825 
14826   format %{ "and  $dst, $src1, $src2\t# int" %}
14827 
14828   ins_cost(INSN_COST);
14829   ins_encode %{
14830     __ andr(as_Register($dst$$reg),
14831             as_Register($src1$$reg),
14832             (uint64_t)($src2$$constant));
14833   %}
14834 
14835   ins_pipe(ialu_reg_imm);
14836 %}
14837 
14838 // Or Instructions
14839 
14840 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14841   match(Set dst (OrL src1 src2));
14842 
14843   format %{ "orr  $dst, $src1, $src2\t# int" %}
14844 
14845   ins_cost(INSN_COST);
14846   ins_encode %{
14847     __ orr(as_Register($dst$$reg),
14848            as_Register($src1$$reg),
14849            as_Register($src2$$reg));
14850   %}
14851 
14852   ins_pipe(ialu_reg_reg);
14853 %}
14854 
14855 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14856   match(Set dst (OrL src1 src2));
14857 
14858   format %{ "orr  $dst, $src1, $src2\t# int" %}
14859 
14860   ins_cost(INSN_COST);
14861   ins_encode %{
14862     __ orr(as_Register($dst$$reg),
14863            as_Register($src1$$reg),
14864            (uint64_t)($src2$$constant));
14865   %}
14866 
14867   ins_pipe(ialu_reg_imm);
14868 %}
14869 
14870 // Xor Instructions
14871 
14872 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14873   match(Set dst (XorL src1 src2));
14874 
14875   format %{ "eor  $dst, $src1, $src2\t# int" %}
14876 
14877   ins_cost(INSN_COST);
14878   ins_encode %{
14879     __ eor(as_Register($dst$$reg),
14880            as_Register($src1$$reg),
14881            as_Register($src2$$reg));
14882   %}
14883 
14884   ins_pipe(ialu_reg_reg);
14885 %}
14886 
14887 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14888   match(Set dst (XorL src1 src2));
14889 
14890   ins_cost(INSN_COST);
14891   format %{ "eor  $dst, $src1, $src2\t# int" %}
14892 
14893   ins_encode %{
14894     __ eor(as_Register($dst$$reg),
14895            as_Register($src1$$reg),
14896            (uint64_t)($src2$$constant));
14897   %}
14898 
14899   ins_pipe(ialu_reg_imm);
14900 %}
14901 
14902 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14903 %{
14904   match(Set dst (ConvI2L src));
14905 
14906   ins_cost(INSN_COST);
14907   format %{ "sxtw  $dst, $src\t# i2l" %}
14908   ins_encode %{
14909     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14910   %}
14911   ins_pipe(ialu_reg_shift);
14912 %}
14913 
14914 // this pattern occurs in bigmath arithmetic
14915 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14916 %{
14917   match(Set dst (AndL (ConvI2L src) mask));
14918 
14919   ins_cost(INSN_COST);
14920   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14921   ins_encode %{
14922     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14923   %}
14924 
14925   ins_pipe(ialu_reg_shift);
14926 %}
14927 
14928 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14929   match(Set dst (ConvL2I src));
14930 
14931   ins_cost(INSN_COST);
14932   format %{ "movw  $dst, $src \t// l2i" %}
14933 
14934   ins_encode %{
14935     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14936   %}
14937 
14938   ins_pipe(ialu_reg);
14939 %}
14940 
14941 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14942 %{
14943   match(Set dst (Conv2B src));
14944   effect(KILL cr);
14945 
14946   format %{
14947     "cmpw $src, zr\n\t"
14948     "cset $dst, ne"
14949   %}
14950 
14951   ins_encode %{
14952     __ cmpw(as_Register($src$$reg), zr);
14953     __ cset(as_Register($dst$$reg), Assembler::NE);
14954   %}
14955 
14956   ins_pipe(ialu_reg);
14957 %}
14958 
14959 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14960 %{
14961   match(Set dst (Conv2B src));
14962   effect(KILL cr);
14963 
14964   format %{
14965     "cmp  $src, zr\n\t"
14966     "cset $dst, ne"
14967   %}
14968 
14969   ins_encode %{
14970     __ cmp(as_Register($src$$reg), zr);
14971     __ cset(as_Register($dst$$reg), Assembler::NE);
14972   %}
14973 
14974   ins_pipe(ialu_reg);
14975 %}
14976 
14977 instruct convD2F_reg(vRegF dst, vRegD src) %{
14978   match(Set dst (ConvD2F src));
14979 
14980   ins_cost(INSN_COST * 5);
14981   format %{ "fcvtd  $dst, $src \t// d2f" %}
14982 
14983   ins_encode %{
14984     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14985   %}
14986 
14987   ins_pipe(fp_d2f);
14988 %}
14989 
14990 instruct convF2D_reg(vRegD dst, vRegF src) %{
14991   match(Set dst (ConvF2D src));
14992 
14993   ins_cost(INSN_COST * 5);
14994   format %{ "fcvts  $dst, $src \t// f2d" %}
14995 
14996   ins_encode %{
14997     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14998   %}
14999 
15000   ins_pipe(fp_f2d);
15001 %}
15002 
15003 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15004   match(Set dst (ConvF2I src));
15005 
15006   ins_cost(INSN_COST * 5);
15007   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
15008 
15009   ins_encode %{
15010     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15011   %}
15012 
15013   ins_pipe(fp_f2i);
15014 %}
15015 
15016 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15017   match(Set dst (ConvF2L src));
15018 
15019   ins_cost(INSN_COST * 5);
15020   format %{ "fcvtzs  $dst, $src \t// f2l" %}
15021 
15022   ins_encode %{
15023     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15024   %}
15025 
15026   ins_pipe(fp_f2l);
15027 %}
15028 
15029 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15030   match(Set dst (ConvI2F src));
15031 
15032   ins_cost(INSN_COST * 5);
15033   format %{ "scvtfws  $dst, $src \t// i2f" %}
15034 
15035   ins_encode %{
15036     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15037   %}
15038 
15039   ins_pipe(fp_i2f);
15040 %}
15041 
15042 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15043   match(Set dst (ConvL2F src));
15044 
15045   ins_cost(INSN_COST * 5);
15046   format %{ "scvtfs  $dst, $src \t// l2f" %}
15047 
15048   ins_encode %{
15049     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15050   %}
15051 
15052   ins_pipe(fp_l2f);
15053 %}
15054 
15055 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15056   match(Set dst (ConvD2I src));
15057 
15058   ins_cost(INSN_COST * 5);
15059   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15060 
15061   ins_encode %{
15062     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15063   %}
15064 
15065   ins_pipe(fp_d2i);
15066 %}
15067 
15068 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15069   match(Set dst (ConvD2L src));
15070 
15071   ins_cost(INSN_COST * 5);
15072   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15073 
15074   ins_encode %{
15075     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15076   %}
15077 
15078   ins_pipe(fp_d2l);
15079 %}
15080 
15081 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15082   match(Set dst (ConvI2D src));
15083 
15084   ins_cost(INSN_COST * 5);
15085   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15086 
15087   ins_encode %{
15088     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15089   %}
15090 
15091   ins_pipe(fp_i2d);
15092 %}
15093 
15094 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15095   match(Set dst (ConvL2D src));
15096 
15097   ins_cost(INSN_COST * 5);
15098   format %{ "scvtfd  $dst, $src \t// l2d" %}
15099 
15100   ins_encode %{
15101     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15102   %}
15103 
15104   ins_pipe(fp_l2d);
15105 %}
15106 
15107 // stack <-> reg and reg <-> reg shuffles with no conversion
15108 
15109 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15110 
15111   match(Set dst (MoveF2I src));
15112 
15113   effect(DEF dst, USE src);
15114 
15115   ins_cost(4 * INSN_COST);
15116 
15117   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15118 
15119   ins_encode %{
15120     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15121   %}
15122 
15123   ins_pipe(iload_reg_reg);
15124 
15125 %}
15126 
15127 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15128 
15129   match(Set dst (MoveI2F src));
15130 
15131   effect(DEF dst, USE src);
15132 
15133   ins_cost(4 * INSN_COST);
15134 
15135   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15136 
15137   ins_encode %{
15138     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15139   %}
15140 
15141   ins_pipe(pipe_class_memory);
15142 
15143 %}
15144 
15145 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15146 
15147   match(Set dst (MoveD2L src));
15148 
15149   effect(DEF dst, USE src);
15150 
15151   ins_cost(4 * INSN_COST);
15152 
15153   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15154 
15155   ins_encode %{
15156     __ ldr($dst$$Register, Address(sp, $src$$disp));
15157   %}
15158 
15159   ins_pipe(iload_reg_reg);
15160 
15161 %}
15162 
15163 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15164 
15165   match(Set dst (MoveL2D src));
15166 
15167   effect(DEF dst, USE src);
15168 
15169   ins_cost(4 * INSN_COST);
15170 
15171   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15172 
15173   ins_encode %{
15174     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15175   %}
15176 
15177   ins_pipe(pipe_class_memory);
15178 
15179 %}
15180 
15181 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15182 
15183   match(Set dst (MoveF2I src));
15184 
15185   effect(DEF dst, USE src);
15186 
15187   ins_cost(INSN_COST);
15188 
15189   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15190 
15191   ins_encode %{
15192     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15193   %}
15194 
15195   ins_pipe(pipe_class_memory);
15196 
15197 %}
15198 
15199 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15200 
15201   match(Set dst (MoveI2F src));
15202 
15203   effect(DEF dst, USE src);
15204 
15205   ins_cost(INSN_COST);
15206 
15207   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15208 
15209   ins_encode %{
15210     __ strw($src$$Register, Address(sp, $dst$$disp));
15211   %}
15212 
15213   ins_pipe(istore_reg_reg);
15214 
15215 %}
15216 
15217 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15218 
15219   match(Set dst (MoveD2L src));
15220 
15221   effect(DEF dst, USE src);
15222 
15223   ins_cost(INSN_COST);
15224 
15225   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15226 
15227   ins_encode %{
15228     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15229   %}
15230 
15231   ins_pipe(pipe_class_memory);
15232 
15233 %}
15234 
15235 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15236 
15237   match(Set dst (MoveL2D src));
15238 
15239   effect(DEF dst, USE src);
15240 
15241   ins_cost(INSN_COST);
15242 
15243   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15244 
15245   ins_encode %{
15246     __ str($src$$Register, Address(sp, $dst$$disp));
15247   %}
15248 
15249   ins_pipe(istore_reg_reg);
15250 
15251 %}
15252 
15253 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15254 
15255   match(Set dst (MoveF2I src));
15256 
15257   effect(DEF dst, USE src);
15258 
15259   ins_cost(INSN_COST);
15260 
15261   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15262 
15263   ins_encode %{
15264     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15265   %}
15266 
15267   ins_pipe(fp_f2i);
15268 
15269 %}
15270 
15271 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15272 
15273   match(Set dst (MoveI2F src));
15274 
15275   effect(DEF dst, USE src);
15276 
15277   ins_cost(INSN_COST);
15278 
15279   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15280 
15281   ins_encode %{
15282     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15283   %}
15284 
15285   ins_pipe(fp_i2f);
15286 
15287 %}
15288 
15289 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15290 
15291   match(Set dst (MoveD2L src));
15292 
15293   effect(DEF dst, USE src);
15294 
15295   ins_cost(INSN_COST);
15296 
15297   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15298 
15299   ins_encode %{
15300     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15301   %}
15302 
15303   ins_pipe(fp_d2l);
15304 
15305 %}
15306 
15307 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15308 
15309   match(Set dst (MoveL2D src));
15310 
15311   effect(DEF dst, USE src);
15312 
15313   ins_cost(INSN_COST);
15314 
15315   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15316 
15317   ins_encode %{
15318     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15319   %}
15320 
15321   ins_pipe(fp_l2d);
15322 
15323 %}
15324 
15325 // ============================================================================
15326 // clearing of an array
15327 
15328 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15329 %{
15330   match(Set dummy (ClearArray (Binary cnt base) zero));
15331   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15332 
15333   ins_cost(4 * INSN_COST);
15334   format %{ "ClearArray $cnt, $base" %}
15335 
15336   ins_encode %{
15337     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15338     if (tpc == NULL) {
15339       ciEnv::current()->record_failure("CodeCache is full");
15340       return;
15341     }
15342   %}
15343 
15344   ins_pipe(pipe_class_memory);
15345 %}
15346 
15347 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15348 %{
15349   predicate(((ClearArrayNode*)n)->word_copy_only());
15350   match(Set dummy (ClearArray (Binary cnt base) val));
15351   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15352 
15353   ins_cost(4 * INSN_COST);
15354   format %{ "ClearArray $cnt, $base, $val" %}
15355 
15356   ins_encode %{
15357     __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15358   %}
15359 
15360   ins_pipe(pipe_class_memory);
15361 %}
15362 
15363 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15364 %{
15365   predicate((uint64_t)n->in(2)->get_long()
15366             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15367             && !((ClearArrayNode*)n)->word_copy_only());
15368   match(Set dummy (ClearArray cnt base));
15369   effect(TEMP temp, USE_KILL base, KILL cr);
15370 
15371   ins_cost(4 * INSN_COST);
15372   format %{ "ClearArray $cnt, $base" %}
15373 
15374   ins_encode %{
15375     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15376   %}
15377 
15378   ins_pipe(pipe_class_memory);
15379 %}
15380 
15381 // ============================================================================
15382 // Overflow Math Instructions
15383 
15384 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15385 %{
15386   match(Set cr (OverflowAddI op1 op2));
15387 
15388   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15389   ins_cost(INSN_COST);
15390   ins_encode %{
15391     __ cmnw($op1$$Register, $op2$$Register);
15392   %}
15393 
15394   ins_pipe(icmp_reg_reg);
15395 %}
15396 
15397 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15398 %{
15399   match(Set cr (OverflowAddI op1 op2));
15400 
15401   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15402   ins_cost(INSN_COST);
15403   ins_encode %{
15404     __ cmnw($op1$$Register, $op2$$constant);
15405   %}
15406 
15407   ins_pipe(icmp_reg_imm);
15408 %}
15409 
15410 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15411 %{
15412   match(Set cr (OverflowAddL op1 op2));
15413 
15414   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15415   ins_cost(INSN_COST);
15416   ins_encode %{
15417     __ cmn($op1$$Register, $op2$$Register);
15418   %}
15419 
15420   ins_pipe(icmp_reg_reg);
15421 %}
15422 
15423 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15424 %{
15425   match(Set cr (OverflowAddL op1 op2));
15426 
15427   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15428   ins_cost(INSN_COST);
15429   ins_encode %{
15430     __ cmn($op1$$Register, $op2$$constant);
15431   %}
15432 
15433   ins_pipe(icmp_reg_imm);
15434 %}
15435 
15436 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15437 %{
15438   match(Set cr (OverflowSubI op1 op2));
15439 
15440   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15441   ins_cost(INSN_COST);
15442   ins_encode %{
15443     __ cmpw($op1$$Register, $op2$$Register);
15444   %}
15445 
15446   ins_pipe(icmp_reg_reg);
15447 %}
15448 
15449 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15450 %{
15451   match(Set cr (OverflowSubI op1 op2));
15452 
15453   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15454   ins_cost(INSN_COST);
15455   ins_encode %{
15456     __ cmpw($op1$$Register, $op2$$constant);
15457   %}
15458 
15459   ins_pipe(icmp_reg_imm);
15460 %}
15461 
15462 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15463 %{
15464   match(Set cr (OverflowSubL op1 op2));
15465 
15466   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15467   ins_cost(INSN_COST);
15468   ins_encode %{
15469     __ cmp($op1$$Register, $op2$$Register);
15470   %}
15471 
15472   ins_pipe(icmp_reg_reg);
15473 %}
15474 
15475 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15476 %{
15477   match(Set cr (OverflowSubL op1 op2));
15478 
15479   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15480   ins_cost(INSN_COST);
15481   ins_encode %{
15482     __ subs(zr, $op1$$Register, $op2$$constant);
15483   %}
15484 
15485   ins_pipe(icmp_reg_imm);
15486 %}
15487 
15488 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15489 %{
15490   match(Set cr (OverflowSubI zero op1));
15491 
15492   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15493   ins_cost(INSN_COST);
15494   ins_encode %{
15495     __ cmpw(zr, $op1$$Register);
15496   %}
15497 
15498   ins_pipe(icmp_reg_imm);
15499 %}
15500 
15501 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15502 %{
15503   match(Set cr (OverflowSubL zero op1));
15504 
15505   format %{ "cmp   zr, $op1\t# overflow check long" %}
15506   ins_cost(INSN_COST);
15507   ins_encode %{
15508     __ cmp(zr, $op1$$Register);
15509   %}
15510 
15511   ins_pipe(icmp_reg_imm);
15512 %}
15513 
15514 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15515 %{
15516   match(Set cr (OverflowMulI op1 op2));
15517 
15518   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15519             "cmp   rscratch1, rscratch1, sxtw\n\t"
15520             "movw  rscratch1, #0x80000000\n\t"
15521             "cselw rscratch1, rscratch1, zr, NE\n\t"
15522             "cmpw  rscratch1, #1" %}
15523   ins_cost(5 * INSN_COST);
15524   ins_encode %{
15525     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15526     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15527     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15528     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15529     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15530   %}
15531 
15532   ins_pipe(pipe_slow);
15533 %}
15534 
15535 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15536 %{
15537   match(If cmp (OverflowMulI op1 op2));
15538   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15539             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15540   effect(USE labl, KILL cr);
15541 
15542   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15543             "cmp   rscratch1, rscratch1, sxtw\n\t"
15544             "b$cmp   $labl" %}
15545   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15546   ins_encode %{
15547     Label* L = $labl$$label;
15548     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15549     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15550     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15551     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15552   %}
15553 
15554   ins_pipe(pipe_serial);
15555 %}
15556 
15557 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15558 %{
15559   match(Set cr (OverflowMulL op1 op2));
15560 
15561   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15562             "smulh rscratch2, $op1, $op2\n\t"
15563             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15564             "movw  rscratch1, #0x80000000\n\t"
15565             "cselw rscratch1, rscratch1, zr, NE\n\t"
15566             "cmpw  rscratch1, #1" %}
15567   ins_cost(6 * INSN_COST);
15568   ins_encode %{
15569     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15570     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15571     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15572     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15573     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15574     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15575   %}
15576 
15577   ins_pipe(pipe_slow);
15578 %}
15579 
15580 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15581 %{
15582   match(If cmp (OverflowMulL op1 op2));
15583   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15584             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15585   effect(USE labl, KILL cr);
15586 
15587   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15588             "smulh rscratch2, $op1, $op2\n\t"
15589             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15590             "b$cmp $labl" %}
15591   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15592   ins_encode %{
15593     Label* L = $labl$$label;
15594     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15595     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15596     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15597     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15598     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15599   %}
15600 
15601   ins_pipe(pipe_serial);
15602 %}
15603 
15604 // ============================================================================
15605 // Compare Instructions
15606 
15607 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15608 %{
15609   match(Set cr (CmpI op1 op2));
15610 
15611   effect(DEF cr, USE op1, USE op2);
15612 
15613   ins_cost(INSN_COST);
15614   format %{ "cmpw  $op1, $op2" %}
15615 
15616   ins_encode(aarch64_enc_cmpw(op1, op2));
15617 
15618   ins_pipe(icmp_reg_reg);
15619 %}
15620 
15621 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15622 %{
15623   match(Set cr (CmpI op1 zero));
15624 
15625   effect(DEF cr, USE op1);
15626 
15627   ins_cost(INSN_COST);
15628   format %{ "cmpw $op1, 0" %}
15629 
15630   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15631 
15632   ins_pipe(icmp_reg_imm);
15633 %}
15634 
15635 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15636 %{
15637   match(Set cr (CmpI op1 op2));
15638 
15639   effect(DEF cr, USE op1);
15640 
15641   ins_cost(INSN_COST);
15642   format %{ "cmpw  $op1, $op2" %}
15643 
15644   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15645 
15646   ins_pipe(icmp_reg_imm);
15647 %}
15648 
15649 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15650 %{
15651   match(Set cr (CmpI op1 op2));
15652 
15653   effect(DEF cr, USE op1);
15654 
15655   ins_cost(INSN_COST * 2);
15656   format %{ "cmpw  $op1, $op2" %}
15657 
15658   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15659 
15660   ins_pipe(icmp_reg_imm);
15661 %}
15662 
15663 // Unsigned compare Instructions; really, same as signed compare
15664 // except it should only be used to feed an If or a CMovI which takes a
15665 // cmpOpU.
15666 
15667 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15668 %{
15669   match(Set cr (CmpU op1 op2));
15670 
15671   effect(DEF cr, USE op1, USE op2);
15672 
15673   ins_cost(INSN_COST);
15674   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15675 
15676   ins_encode(aarch64_enc_cmpw(op1, op2));
15677 
15678   ins_pipe(icmp_reg_reg);
15679 %}
15680 
15681 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15682 %{
15683   match(Set cr (CmpU op1 zero));
15684 
15685   effect(DEF cr, USE op1);
15686 
15687   ins_cost(INSN_COST);
15688   format %{ "cmpw $op1, #0\t# unsigned" %}
15689 
15690   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15691 
15692   ins_pipe(icmp_reg_imm);
15693 %}
15694 
15695 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15696 %{
15697   match(Set cr (CmpU op1 op2));
15698 
15699   effect(DEF cr, USE op1);
15700 
15701   ins_cost(INSN_COST);
15702   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15703 
15704   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15705 
15706   ins_pipe(icmp_reg_imm);
15707 %}
15708 
15709 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15710 %{
15711   match(Set cr (CmpU op1 op2));
15712 
15713   effect(DEF cr, USE op1);
15714 
15715   ins_cost(INSN_COST * 2);
15716   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15717 
15718   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15719 
15720   ins_pipe(icmp_reg_imm);
15721 %}
15722 
15723 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15724 %{
15725   match(Set cr (CmpL op1 op2));
15726 
15727   effect(DEF cr, USE op1, USE op2);
15728 
15729   ins_cost(INSN_COST);
15730   format %{ "cmp  $op1, $op2" %}
15731 
15732   ins_encode(aarch64_enc_cmp(op1, op2));
15733 
15734   ins_pipe(icmp_reg_reg);
15735 %}
15736 
15737 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15738 %{
15739   match(Set cr (CmpL op1 zero));
15740 
15741   effect(DEF cr, USE op1);
15742 
15743   ins_cost(INSN_COST);
15744   format %{ "tst  $op1" %}
15745 
15746   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15747 
15748   ins_pipe(icmp_reg_imm);
15749 %}
15750 
15751 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15752 %{
15753   match(Set cr (CmpL op1 op2));
15754 
15755   effect(DEF cr, USE op1);
15756 
15757   ins_cost(INSN_COST);
15758   format %{ "cmp  $op1, $op2" %}
15759 
15760   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15761 
15762   ins_pipe(icmp_reg_imm);
15763 %}
15764 
15765 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15766 %{
15767   match(Set cr (CmpL op1 op2));
15768 
15769   effect(DEF cr, USE op1);
15770 
15771   ins_cost(INSN_COST * 2);
15772   format %{ "cmp  $op1, $op2" %}
15773 
15774   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15775 
15776   ins_pipe(icmp_reg_imm);
15777 %}
15778 
15779 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15780 %{
15781   match(Set cr (CmpUL op1 op2));
15782 
15783   effect(DEF cr, USE op1, USE op2);
15784 
15785   ins_cost(INSN_COST);
15786   format %{ "cmp  $op1, $op2" %}
15787 
15788   ins_encode(aarch64_enc_cmp(op1, op2));
15789 
15790   ins_pipe(icmp_reg_reg);
15791 %}
15792 
15793 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15794 %{
15795   match(Set cr (CmpUL op1 zero));
15796 
15797   effect(DEF cr, USE op1);
15798 
15799   ins_cost(INSN_COST);
15800   format %{ "tst  $op1" %}
15801 
15802   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15803 
15804   ins_pipe(icmp_reg_imm);
15805 %}
15806 
15807 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15808 %{
15809   match(Set cr (CmpUL op1 op2));
15810 
15811   effect(DEF cr, USE op1);
15812 
15813   ins_cost(INSN_COST);
15814   format %{ "cmp  $op1, $op2" %}
15815 
15816   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15817 
15818   ins_pipe(icmp_reg_imm);
15819 %}
15820 
15821 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15822 %{
15823   match(Set cr (CmpUL op1 op2));
15824 
15825   effect(DEF cr, USE op1);
15826 
15827   ins_cost(INSN_COST * 2);
15828   format %{ "cmp  $op1, $op2" %}
15829 
15830   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15831 
15832   ins_pipe(icmp_reg_imm);
15833 %}
15834 
15835 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15836 %{
15837   match(Set cr (CmpP op1 op2));
15838 
15839   effect(DEF cr, USE op1, USE op2);
15840 
15841   ins_cost(INSN_COST);
15842   format %{ "cmp  $op1, $op2\t // ptr" %}
15843 
15844   ins_encode(aarch64_enc_cmpp(op1, op2));
15845 
15846   ins_pipe(icmp_reg_reg);
15847 %}
15848 
15849 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15850 %{
15851   match(Set cr (CmpN op1 op2));
15852 
15853   effect(DEF cr, USE op1, USE op2);
15854 
15855   ins_cost(INSN_COST);
15856   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15857 
15858   ins_encode(aarch64_enc_cmpn(op1, op2));
15859 
15860   ins_pipe(icmp_reg_reg);
15861 %}
15862 
15863 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15864 %{
15865   match(Set cr (CmpP op1 zero));
15866 
15867   effect(DEF cr, USE op1, USE zero);
15868 
15869   ins_cost(INSN_COST);
15870   format %{ "cmp  $op1, 0\t // ptr" %}
15871 
15872   ins_encode(aarch64_enc_testp(op1));
15873 
15874   ins_pipe(icmp_reg_imm);
15875 %}
15876 
15877 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15878 %{
15879   match(Set cr (CmpN op1 zero));
15880 
15881   effect(DEF cr, USE op1, USE zero);
15882 
15883   ins_cost(INSN_COST);
15884   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15885 
15886   ins_encode(aarch64_enc_testn(op1));
15887 
15888   ins_pipe(icmp_reg_imm);
15889 %}
15890 
15891 // FP comparisons
15892 //
15893 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15894 // using normal cmpOp. See declaration of rFlagsReg for details.
15895 
15896 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15897 %{
15898   match(Set cr (CmpF src1 src2));
15899 
15900   ins_cost(3 * INSN_COST);
15901   format %{ "fcmps $src1, $src2" %}
15902 
15903   ins_encode %{
15904     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15905   %}
15906 
15907   ins_pipe(pipe_class_compare);
15908 %}
15909 
15910 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15911 %{
15912   match(Set cr (CmpF src1 src2));
15913 
15914   ins_cost(3 * INSN_COST);
15915   format %{ "fcmps $src1, 0.0" %}
15916 
15917   ins_encode %{
15918     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15919   %}
15920 
15921   ins_pipe(pipe_class_compare);
15922 %}
15923 // FROM HERE
15924 
15925 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15926 %{
15927   match(Set cr (CmpD src1 src2));
15928 
15929   ins_cost(3 * INSN_COST);
15930   format %{ "fcmpd $src1, $src2" %}
15931 
15932   ins_encode %{
15933     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15934   %}
15935 
15936   ins_pipe(pipe_class_compare);
15937 %}
15938 
15939 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15940 %{
15941   match(Set cr (CmpD src1 src2));
15942 
15943   ins_cost(3 * INSN_COST);
15944   format %{ "fcmpd $src1, 0.0" %}
15945 
15946   ins_encode %{
15947     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15948   %}
15949 
15950   ins_pipe(pipe_class_compare);
15951 %}
15952 
15953 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15954 %{
15955   match(Set dst (CmpF3 src1 src2));
15956   effect(KILL cr);
15957 
15958   ins_cost(5 * INSN_COST);
15959   format %{ "fcmps $src1, $src2\n\t"
15960             "csinvw($dst, zr, zr, eq\n\t"
15961             "csnegw($dst, $dst, $dst, lt)"
15962   %}
15963 
15964   ins_encode %{
15965     Label done;
15966     FloatRegister s1 = as_FloatRegister($src1$$reg);
15967     FloatRegister s2 = as_FloatRegister($src2$$reg);
15968     Register d = as_Register($dst$$reg);
15969     __ fcmps(s1, s2);
15970     // installs 0 if EQ else -1
15971     __ csinvw(d, zr, zr, Assembler::EQ);
15972     // keeps -1 if less or unordered else installs 1
15973     __ csnegw(d, d, d, Assembler::LT);
15974     __ bind(done);
15975   %}
15976 
15977   ins_pipe(pipe_class_default);
15978 
15979 %}
15980 
15981 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15982 %{
15983   match(Set dst (CmpD3 src1 src2));
15984   effect(KILL cr);
15985 
15986   ins_cost(5 * INSN_COST);
15987   format %{ "fcmpd $src1, $src2\n\t"
15988             "csinvw($dst, zr, zr, eq\n\t"
15989             "csnegw($dst, $dst, $dst, lt)"
15990   %}
15991 
15992   ins_encode %{
15993     Label done;
15994     FloatRegister s1 = as_FloatRegister($src1$$reg);
15995     FloatRegister s2 = as_FloatRegister($src2$$reg);
15996     Register d = as_Register($dst$$reg);
15997     __ fcmpd(s1, s2);
15998     // installs 0 if EQ else -1
15999     __ csinvw(d, zr, zr, Assembler::EQ);
16000     // keeps -1 if less or unordered else installs 1
16001     __ csnegw(d, d, d, Assembler::LT);
16002     __ bind(done);
16003   %}
16004   ins_pipe(pipe_class_default);
16005 
16006 %}
16007 
16008 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16009 %{
16010   match(Set dst (CmpF3 src1 zero));
16011   effect(KILL cr);
16012 
16013   ins_cost(5 * INSN_COST);
16014   format %{ "fcmps $src1, 0.0\n\t"
16015             "csinvw($dst, zr, zr, eq\n\t"
16016             "csnegw($dst, $dst, $dst, lt)"
16017   %}
16018 
16019   ins_encode %{
16020     Label done;
16021     FloatRegister s1 = as_FloatRegister($src1$$reg);
16022     Register d = as_Register($dst$$reg);
16023     __ fcmps(s1, 0.0);
16024     // installs 0 if EQ else -1
16025     __ csinvw(d, zr, zr, Assembler::EQ);
16026     // keeps -1 if less or unordered else installs 1
16027     __ csnegw(d, d, d, Assembler::LT);
16028     __ bind(done);
16029   %}
16030 
16031   ins_pipe(pipe_class_default);
16032 
16033 %}
16034 
16035 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16036 %{
16037   match(Set dst (CmpD3 src1 zero));
16038   effect(KILL cr);
16039 
16040   ins_cost(5 * INSN_COST);
16041   format %{ "fcmpd $src1, 0.0\n\t"
16042             "csinvw($dst, zr, zr, eq\n\t"
16043             "csnegw($dst, $dst, $dst, lt)"
16044   %}
16045 
16046   ins_encode %{
16047     Label done;
16048     FloatRegister s1 = as_FloatRegister($src1$$reg);
16049     Register d = as_Register($dst$$reg);
16050     __ fcmpd(s1, 0.0);
16051     // installs 0 if EQ else -1
16052     __ csinvw(d, zr, zr, Assembler::EQ);
16053     // keeps -1 if less or unordered else installs 1
16054     __ csnegw(d, d, d, Assembler::LT);
16055     __ bind(done);
16056   %}
16057   ins_pipe(pipe_class_default);
16058 
16059 %}
16060 
16061 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16062 %{
16063   match(Set dst (CmpLTMask p q));
16064   effect(KILL cr);
16065 
16066   ins_cost(3 * INSN_COST);
16067 
16068   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16069             "csetw $dst, lt\n\t"
16070             "subw $dst, zr, $dst"
16071   %}
16072 
16073   ins_encode %{
16074     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16075     __ csetw(as_Register($dst$$reg), Assembler::LT);
16076     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16077   %}
16078 
16079   ins_pipe(ialu_reg_reg);
16080 %}
16081 
16082 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16083 %{
16084   match(Set dst (CmpLTMask src zero));
16085   effect(KILL cr);
16086 
16087   ins_cost(INSN_COST);
16088 
16089   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16090 
16091   ins_encode %{
16092     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16093   %}
16094 
16095   ins_pipe(ialu_reg_shift);
16096 %}
16097 
16098 // ============================================================================
16099 // Max and Min
16100 
16101 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16102 %{
16103   effect( DEF dst, USE src1, USE src2, USE cr );
16104 
16105   ins_cost(INSN_COST * 2);
16106   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
16107 
16108   ins_encode %{
16109     __ cselw(as_Register($dst$$reg),
16110              as_Register($src1$$reg),
16111              as_Register($src2$$reg),
16112              Assembler::LT);
16113   %}
16114 
16115   ins_pipe(icond_reg_reg);
16116 %}
16117 
16118 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16119 %{
16120   match(Set dst (MinI src1 src2));
16121   ins_cost(INSN_COST * 3);
16122 
16123   expand %{
16124     rFlagsReg cr;
16125     compI_reg_reg(cr, src1, src2);
16126     cmovI_reg_reg_lt(dst, src1, src2, cr);
16127   %}
16128 
16129 %}
16130 // FROM HERE
16131 
16132 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16133 %{
16134   effect( DEF dst, USE src1, USE src2, USE cr );
16135 
16136   ins_cost(INSN_COST * 2);
16137   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
16138 
16139   ins_encode %{
16140     __ cselw(as_Register($dst$$reg),
16141              as_Register($src1$$reg),
16142              as_Register($src2$$reg),
16143              Assembler::GT);
16144   %}
16145 
16146   ins_pipe(icond_reg_reg);
16147 %}
16148 
16149 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16150 %{
16151   match(Set dst (MaxI src1 src2));
16152   ins_cost(INSN_COST * 3);
16153   expand %{
16154     rFlagsReg cr;
16155     compI_reg_reg(cr, src1, src2);
16156     cmovI_reg_reg_gt(dst, src1, src2, cr);
16157   %}
16158 %}
16159 
16160 // ============================================================================
16161 // Branch Instructions
16162 
16163 // Direct Branch.
16164 instruct branch(label lbl)
16165 %{
16166   match(Goto);
16167 
16168   effect(USE lbl);
16169 
16170   ins_cost(BRANCH_COST);
16171   format %{ "b  $lbl" %}
16172 
16173   ins_encode(aarch64_enc_b(lbl));
16174 
16175   ins_pipe(pipe_branch);
16176 %}
16177 
16178 // Conditional Near Branch
16179 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16180 %{
16181   // Same match rule as `branchConFar'.
16182   match(If cmp cr);
16183 
16184   effect(USE lbl);
16185 
16186   ins_cost(BRANCH_COST);
16187   // If set to 1 this indicates that the current instruction is a
16188   // short variant of a long branch. This avoids using this
16189   // instruction in first-pass matching. It will then only be used in
16190   // the `Shorten_branches' pass.
16191   // ins_short_branch(1);
16192   format %{ "b$cmp  $lbl" %}
16193 
16194   ins_encode(aarch64_enc_br_con(cmp, lbl));
16195 
16196   ins_pipe(pipe_branch_cond);
16197 %}
16198 
16199 // Conditional Near Branch Unsigned
16200 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16201 %{
16202   // Same match rule as `branchConFar'.
16203   match(If cmp cr);
16204 
16205   effect(USE lbl);
16206 
16207   ins_cost(BRANCH_COST);
16208   // If set to 1 this indicates that the current instruction is a
16209   // short variant of a long branch. This avoids using this
16210   // instruction in first-pass matching. It will then only be used in
16211   // the `Shorten_branches' pass.
16212   // ins_short_branch(1);
16213   format %{ "b$cmp  $lbl\t# unsigned" %}
16214 
16215   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16216 
16217   ins_pipe(pipe_branch_cond);
16218 %}
16219 
16220 // Make use of CBZ and CBNZ.  These instructions, as well as being
16221 // shorter than (cmp; branch), have the additional benefit of not
16222 // killing the flags.
16223 
16224 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16225   match(If cmp (CmpI op1 op2));
16226   effect(USE labl);
16227 
16228   ins_cost(BRANCH_COST);
16229   format %{ "cbw$cmp   $op1, $labl" %}
16230   ins_encode %{
16231     Label* L = $labl$$label;
16232     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16233     if (cond == Assembler::EQ)
16234       __ cbzw($op1$$Register, *L);
16235     else
16236       __ cbnzw($op1$$Register, *L);
16237   %}
16238   ins_pipe(pipe_cmp_branch);
16239 %}
16240 
16241 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16242   match(If cmp (CmpL op1 op2));
16243   effect(USE labl);
16244 
16245   ins_cost(BRANCH_COST);
16246   format %{ "cb$cmp   $op1, $labl" %}
16247   ins_encode %{
16248     Label* L = $labl$$label;
16249     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16250     if (cond == Assembler::EQ)
16251       __ cbz($op1$$Register, *L);
16252     else
16253       __ cbnz($op1$$Register, *L);
16254   %}
16255   ins_pipe(pipe_cmp_branch);
16256 %}
16257 
16258 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16259   match(If cmp (CmpP op1 op2));
16260   effect(USE labl);
16261 
16262   ins_cost(BRANCH_COST);
16263   format %{ "cb$cmp   $op1, $labl" %}
16264   ins_encode %{
16265     Label* L = $labl$$label;
16266     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16267     if (cond == Assembler::EQ)
16268       __ cbz($op1$$Register, *L);
16269     else
16270       __ cbnz($op1$$Register, *L);
16271   %}
16272   ins_pipe(pipe_cmp_branch);
16273 %}
16274 
16275 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16276   match(If cmp (CmpN op1 op2));
16277   effect(USE labl);
16278 
16279   ins_cost(BRANCH_COST);
16280   format %{ "cbw$cmp   $op1, $labl" %}
16281   ins_encode %{
16282     Label* L = $labl$$label;
16283     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16284     if (cond == Assembler::EQ)
16285       __ cbzw($op1$$Register, *L);
16286     else
16287       __ cbnzw($op1$$Register, *L);
16288   %}
16289   ins_pipe(pipe_cmp_branch);
16290 %}
16291 
16292 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16293   match(If cmp (CmpP (DecodeN oop) zero));
16294   effect(USE labl);
16295 
16296   ins_cost(BRANCH_COST);
16297   format %{ "cb$cmp   $oop, $labl" %}
16298   ins_encode %{
16299     Label* L = $labl$$label;
16300     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16301     if (cond == Assembler::EQ)
16302       __ cbzw($oop$$Register, *L);
16303     else
16304       __ cbnzw($oop$$Register, *L);
16305   %}
16306   ins_pipe(pipe_cmp_branch);
16307 %}
16308 
16309 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16310   match(If cmp (CmpU op1 op2));
16311   effect(USE labl);
16312 
16313   ins_cost(BRANCH_COST);
16314   format %{ "cbw$cmp   $op1, $labl" %}
16315   ins_encode %{
16316     Label* L = $labl$$label;
16317     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16318     if (cond == Assembler::EQ || cond == Assembler::LS)
16319       __ cbzw($op1$$Register, *L);
16320     else
16321       __ cbnzw($op1$$Register, *L);
16322   %}
16323   ins_pipe(pipe_cmp_branch);
16324 %}
16325 
16326 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16327   match(If cmp (CmpUL op1 op2));
16328   effect(USE labl);
16329 
16330   ins_cost(BRANCH_COST);
16331   format %{ "cb$cmp   $op1, $labl" %}
16332   ins_encode %{
16333     Label* L = $labl$$label;
16334     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16335     if (cond == Assembler::EQ || cond == Assembler::LS)
16336       __ cbz($op1$$Register, *L);
16337     else
16338       __ cbnz($op1$$Register, *L);
16339   %}
16340   ins_pipe(pipe_cmp_branch);
16341 %}
16342 
16343 // Test bit and Branch
16344 
16345 // Patterns for short (< 32KiB) variants
16346 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16347   match(If cmp (CmpL op1 op2));
16348   effect(USE labl);
16349 
16350   ins_cost(BRANCH_COST);
16351   format %{ "cb$cmp   $op1, $labl # long" %}
16352   ins_encode %{
16353     Label* L = $labl$$label;
16354     Assembler::Condition cond =
16355       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16356     __ tbr(cond, $op1$$Register, 63, *L);
16357   %}
16358   ins_pipe(pipe_cmp_branch);
16359   ins_short_branch(1);
16360 %}
16361 
16362 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16363   match(If cmp (CmpI op1 op2));
16364   effect(USE labl);
16365 
16366   ins_cost(BRANCH_COST);
16367   format %{ "cb$cmp   $op1, $labl # int" %}
16368   ins_encode %{
16369     Label* L = $labl$$label;
16370     Assembler::Condition cond =
16371       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16372     __ tbr(cond, $op1$$Register, 31, *L);
16373   %}
16374   ins_pipe(pipe_cmp_branch);
16375   ins_short_branch(1);
16376 %}
16377 
16378 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16379   match(If cmp (CmpL (AndL op1 op2) op3));
16380   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16381   effect(USE labl);
16382 
16383   ins_cost(BRANCH_COST);
16384   format %{ "tb$cmp   $op1, $op2, $labl" %}
16385   ins_encode %{
16386     Label* L = $labl$$label;
16387     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16388     int bit = exact_log2_long($op2$$constant);
16389     __ tbr(cond, $op1$$Register, bit, *L);
16390   %}
16391   ins_pipe(pipe_cmp_branch);
16392   ins_short_branch(1);
16393 %}
16394 
16395 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16396   match(If cmp (CmpI (AndI op1 op2) op3));
16397   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16398   effect(USE labl);
16399 
16400   ins_cost(BRANCH_COST);
16401   format %{ "tb$cmp   $op1, $op2, $labl" %}
16402   ins_encode %{
16403     Label* L = $labl$$label;
16404     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16405     int bit = exact_log2((juint)$op2$$constant);
16406     __ tbr(cond, $op1$$Register, bit, *L);
16407   %}
16408   ins_pipe(pipe_cmp_branch);
16409   ins_short_branch(1);
16410 %}
16411 
16412 // And far variants
16413 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16414   match(If cmp (CmpL op1 op2));
16415   effect(USE labl);
16416 
16417   ins_cost(BRANCH_COST);
16418   format %{ "cb$cmp   $op1, $labl # long" %}
16419   ins_encode %{
16420     Label* L = $labl$$label;
16421     Assembler::Condition cond =
16422       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16423     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16424   %}
16425   ins_pipe(pipe_cmp_branch);
16426 %}
16427 
16428 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16429   match(If cmp (CmpI op1 op2));
16430   effect(USE labl);
16431 
16432   ins_cost(BRANCH_COST);
16433   format %{ "cb$cmp   $op1, $labl # int" %}
16434   ins_encode %{
16435     Label* L = $labl$$label;
16436     Assembler::Condition cond =
16437       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16438     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16439   %}
16440   ins_pipe(pipe_cmp_branch);
16441 %}
16442 
16443 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16444   match(If cmp (CmpL (AndL op1 op2) op3));
16445   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16446   effect(USE labl);
16447 
16448   ins_cost(BRANCH_COST);
16449   format %{ "tb$cmp   $op1, $op2, $labl" %}
16450   ins_encode %{
16451     Label* L = $labl$$label;
16452     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16453     int bit = exact_log2_long($op2$$constant);
16454     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16455   %}
16456   ins_pipe(pipe_cmp_branch);
16457 %}
16458 
16459 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16460   match(If cmp (CmpI (AndI op1 op2) op3));
16461   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16462   effect(USE labl);
16463 
16464   ins_cost(BRANCH_COST);
16465   format %{ "tb$cmp   $op1, $op2, $labl" %}
16466   ins_encode %{
16467     Label* L = $labl$$label;
16468     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16469     int bit = exact_log2((juint)$op2$$constant);
16470     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16471   %}
16472   ins_pipe(pipe_cmp_branch);
16473 %}
16474 
16475 // Test bits
16476 
16477 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16478   match(Set cr (CmpL (AndL op1 op2) op3));
16479   predicate(Assembler::operand_valid_for_logical_immediate
16480             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16481 
16482   ins_cost(INSN_COST);
16483   format %{ "tst $op1, $op2 # long" %}
16484   ins_encode %{
16485     __ tst($op1$$Register, $op2$$constant);
16486   %}
16487   ins_pipe(ialu_reg_reg);
16488 %}
16489 
16490 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16491   match(Set cr (CmpI (AndI op1 op2) op3));
16492   predicate(Assembler::operand_valid_for_logical_immediate
16493             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16494 
16495   ins_cost(INSN_COST);
16496   format %{ "tst $op1, $op2 # int" %}
16497   ins_encode %{
16498     __ tstw($op1$$Register, $op2$$constant);
16499   %}
16500   ins_pipe(ialu_reg_reg);
16501 %}
16502 
16503 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16504   match(Set cr (CmpL (AndL op1 op2) op3));
16505 
16506   ins_cost(INSN_COST);
16507   format %{ "tst $op1, $op2 # long" %}
16508   ins_encode %{
16509     __ tst($op1$$Register, $op2$$Register);
16510   %}
16511   ins_pipe(ialu_reg_reg);
16512 %}
16513 
16514 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16515   match(Set cr (CmpI (AndI op1 op2) op3));
16516 
16517   ins_cost(INSN_COST);
16518   format %{ "tstw $op1, $op2 # int" %}
16519   ins_encode %{
16520     __ tstw($op1$$Register, $op2$$Register);
16521   %}
16522   ins_pipe(ialu_reg_reg);
16523 %}
16524 
16525 
16526 // Conditional Far Branch
16527 // Conditional Far Branch Unsigned
16528 // TODO: fixme
16529 
16530 // counted loop end branch near
16531 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16532 %{
16533   match(CountedLoopEnd cmp cr);
16534 
16535   effect(USE lbl);
16536 
16537   ins_cost(BRANCH_COST);
16538   // short variant.
16539   // ins_short_branch(1);
16540   format %{ "b$cmp $lbl \t// counted loop end" %}
16541 
16542   ins_encode(aarch64_enc_br_con(cmp, lbl));
16543 
16544   ins_pipe(pipe_branch);
16545 %}
16546 
16547 // counted loop end branch near Unsigned
16548 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16549 %{
16550   match(CountedLoopEnd cmp cr);
16551 
16552   effect(USE lbl);
16553 
16554   ins_cost(BRANCH_COST);
16555   // short variant.
16556   // ins_short_branch(1);
16557   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16558 
16559   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16560 
16561   ins_pipe(pipe_branch);
16562 %}
16563 
16564 // counted loop end branch far
16565 // counted loop end branch far unsigned
16566 // TODO: fixme
16567 
16568 // ============================================================================
16569 // inlined locking and unlocking
16570 
16571 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16572 %{
16573   match(Set cr (FastLock object box));
16574   effect(TEMP tmp, TEMP tmp2);
16575 
16576   // TODO
16577   // identify correct cost
16578   ins_cost(5 * INSN_COST);
16579   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16580 
16581   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16582 
16583   ins_pipe(pipe_serial);
16584 %}
16585 
16586 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16587 %{
16588   match(Set cr (FastUnlock object box));
16589   effect(TEMP tmp, TEMP tmp2);
16590 
16591   ins_cost(5 * INSN_COST);
16592   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16593 
16594   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16595 
16596   ins_pipe(pipe_serial);
16597 %}
16598 
16599 
16600 // ============================================================================
16601 // Safepoint Instructions
16602 
16603 // TODO
16604 // provide a near and far version of this code
16605 
16606 instruct safePoint(rFlagsReg cr, iRegP poll)
16607 %{
16608   match(SafePoint poll);
16609   effect(KILL cr);
16610 
16611   format %{
16612     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16613   %}
16614   ins_encode %{
16615     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16616   %}
16617   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16618 %}
16619 
16620 
16621 // ============================================================================
16622 // Procedure Call/Return Instructions
16623 
16624 // Call Java Static Instruction
16625 
16626 instruct CallStaticJavaDirect(method meth)
16627 %{
16628   match(CallStaticJava);
16629 
16630   effect(USE meth);
16631 
16632   ins_cost(CALL_COST);
16633 
16634   format %{ "call,static $meth \t// ==> " %}
16635 
16636   ins_encode(aarch64_enc_java_static_call(meth),
16637              aarch64_enc_call_epilog);
16638 
16639   ins_pipe(pipe_class_call);
16640 %}
16641 
16642 // TO HERE
16643 
16644 // Call Java Dynamic Instruction
16645 instruct CallDynamicJavaDirect(method meth)
16646 %{
16647   match(CallDynamicJava);
16648 
16649   effect(USE meth);
16650 
16651   ins_cost(CALL_COST);
16652 
16653   format %{ "CALL,dynamic $meth \t// ==> " %}
16654 
16655   ins_encode(aarch64_enc_java_dynamic_call(meth),
16656              aarch64_enc_call_epilog);
16657 
16658   ins_pipe(pipe_class_call);
16659 %}
16660 
16661 // Call Runtime Instruction
16662 
16663 instruct CallRuntimeDirect(method meth)
16664 %{
16665   match(CallRuntime);
16666 
16667   effect(USE meth);
16668 
16669   ins_cost(CALL_COST);
16670 
16671   format %{ "CALL, runtime $meth" %}
16672 
16673   ins_encode( aarch64_enc_java_to_runtime(meth) );
16674 
16675   ins_pipe(pipe_class_call);
16676 %}
16677 
16678 // Call Runtime Instruction
16679 
16680 instruct CallLeafDirect(method meth)
16681 %{
16682   match(CallLeaf);
16683 
16684   effect(USE meth);
16685 
16686   ins_cost(CALL_COST);
16687 
16688   format %{ "CALL, runtime leaf $meth" %}
16689 
16690   ins_encode( aarch64_enc_java_to_runtime(meth) );
16691 
16692   ins_pipe(pipe_class_call);
16693 %}
16694 
16695 // Call Runtime Instruction
16696 
16697 // entry point is null, target holds the address to call
16698 instruct CallLeafNoFPIndirect(iRegP target)
16699 %{
16700   predicate(n->as_Call()->entry_point() == NULL);
16701 
16702   match(CallLeafNoFP target);
16703 
16704   ins_cost(CALL_COST);
16705 
16706   format %{ "CALL, runtime leaf nofp indirect $target" %}
16707 
16708   ins_encode %{
16709     __ blr($target$$Register);
16710   %}
16711 
16712   ins_pipe(pipe_class_call);
16713 %}
16714 
16715 instruct CallLeafNoFPDirect(method meth)
16716 %{
16717   predicate(n->as_Call()->entry_point() != NULL);
16718 
16719   match(CallLeafNoFP);
16720 
16721   effect(USE meth);
16722 
16723   ins_cost(CALL_COST);
16724 
16725   format %{ "CALL, runtime leaf nofp $meth" %}
16726 
16727   ins_encode( aarch64_enc_java_to_runtime(meth) );
16728 
16729   ins_pipe(pipe_class_call);
16730 %}
16731 
16732 instruct CallNativeDirect(method meth)
16733 %{
16734   match(CallNative);
16735 
16736   effect(USE meth);
16737 
16738   ins_cost(CALL_COST);
16739 
16740   format %{ "CALL, native $meth" %}
16741 
16742   ins_encode( aarch64_enc_java_to_runtime(meth) );
16743 
16744   ins_pipe(pipe_class_call);
16745 %}
16746 
16747 // Tail Call; Jump from runtime stub to Java code.
16748 // Also known as an 'interprocedural jump'.
16749 // Target of jump will eventually return to caller.
16750 // TailJump below removes the return address.
16751 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16752 %{
16753   match(TailCall jump_target method_ptr);
16754 
16755   ins_cost(CALL_COST);
16756 
16757   format %{ "br $jump_target\t# $method_ptr holds method" %}
16758 
16759   ins_encode(aarch64_enc_tail_call(jump_target));
16760 
16761   ins_pipe(pipe_class_call);
16762 %}
16763 
16764 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16765 %{
16766   match(TailJump jump_target ex_oop);
16767 
16768   ins_cost(CALL_COST);
16769 
16770   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16771 
16772   ins_encode(aarch64_enc_tail_jmp(jump_target));
16773 
16774   ins_pipe(pipe_class_call);
16775 %}
16776 
16777 // Create exception oop: created by stack-crawling runtime code.
16778 // Created exception is now available to this handler, and is setup
16779 // just prior to jumping to this handler. No code emitted.
16780 // TODO check
16781 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16782 instruct CreateException(iRegP_R0 ex_oop)
16783 %{
16784   match(Set ex_oop (CreateEx));
16785 
16786   format %{ " -- \t// exception oop; no code emitted" %}
16787 
16788   size(0);
16789 
16790   ins_encode( /*empty*/ );
16791 
16792   ins_pipe(pipe_class_empty);
16793 %}
16794 
16795 // Rethrow exception: The exception oop will come in the first
16796 // argument position. Then JUMP (not call) to the rethrow stub code.
16797 instruct RethrowException() %{
16798   match(Rethrow);
16799   ins_cost(CALL_COST);
16800 
16801   format %{ "b rethrow_stub" %}
16802 
16803   ins_encode( aarch64_enc_rethrow() );
16804 
16805   ins_pipe(pipe_class_call);
16806 %}
16807 
16808 
16809 // Return Instruction
16810 // epilog node loads ret address into lr as part of frame pop
16811 instruct Ret()
16812 %{
16813   match(Return);
16814 
16815   format %{ "ret\t// return register" %}
16816 
16817   ins_encode( aarch64_enc_ret() );
16818 
16819   ins_pipe(pipe_branch);
16820 %}
16821 
16822 // Die now.
16823 instruct ShouldNotReachHere() %{
16824   match(Halt);
16825 
16826   ins_cost(CALL_COST);
16827   format %{ "ShouldNotReachHere" %}
16828 
16829   ins_encode %{
16830     if (is_reachable()) {
16831       __ stop(_halt_reason);
16832     }
16833   %}
16834 
16835   ins_pipe(pipe_class_default);
16836 %}
16837 
16838 // ============================================================================
16839 // Partial Subtype Check
16840 //
16841 // superklass array for an instance of the superklass.  Set a hidden
16842 // internal cache on a hit (cache is checked with exposed code in
16843 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16844 // encoding ALSO sets flags.
16845 
16846 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16847 %{
16848   match(Set result (PartialSubtypeCheck sub super));
16849   effect(KILL cr, KILL temp);
16850 
16851   ins_cost(1100);  // slightly larger than the next version
16852   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16853 
16854   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16855 
16856   opcode(0x1); // Force zero of result reg on hit
16857 
16858   ins_pipe(pipe_class_memory);
16859 %}
16860 
16861 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16862 %{
16863   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16864   effect(KILL temp, KILL result);
16865 
16866   ins_cost(1100);  // slightly larger than the next version
16867   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16868 
16869   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16870 
16871   opcode(0x0); // Don't zero result reg on hit
16872 
16873   ins_pipe(pipe_class_memory);
16874 %}
16875 
16876 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16877                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16878 %{
16879   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16880   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16881   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16882 
16883   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16884   ins_encode %{
16885     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16886     __ string_compare($str1$$Register, $str2$$Register,
16887                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16888                       $tmp1$$Register, $tmp2$$Register,
16889                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16890   %}
16891   ins_pipe(pipe_class_memory);
16892 %}
16893 
16894 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16895                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16896 %{
16897   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16898   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16899   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16900 
16901   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16902   ins_encode %{
16903     __ string_compare($str1$$Register, $str2$$Register,
16904                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16905                       $tmp1$$Register, $tmp2$$Register,
16906                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16907   %}
16908   ins_pipe(pipe_class_memory);
16909 %}
16910 
16911 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16912                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16913                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16914 %{
16915   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16916   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16917   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16918          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16919 
16920   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16921   ins_encode %{
16922     __ string_compare($str1$$Register, $str2$$Register,
16923                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16924                       $tmp1$$Register, $tmp2$$Register,
16925                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16926                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16927   %}
16928   ins_pipe(pipe_class_memory);
16929 %}
16930 
16931 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16932                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16933                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16934 %{
16935   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16936   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16937   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16938          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16939 
16940   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16941   ins_encode %{
16942     __ string_compare($str1$$Register, $str2$$Register,
16943                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16944                       $tmp1$$Register, $tmp2$$Register,
16945                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16946                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16947   %}
16948   ins_pipe(pipe_class_memory);
16949 %}
16950 
16951 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16952        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16953        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16954 %{
16955   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16956   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16957   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16958          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16959   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16960 
16961   ins_encode %{
16962     __ string_indexof($str1$$Register, $str2$$Register,
16963                       $cnt1$$Register, $cnt2$$Register,
16964                       $tmp1$$Register, $tmp2$$Register,
16965                       $tmp3$$Register, $tmp4$$Register,
16966                       $tmp5$$Register, $tmp6$$Register,
16967                       -1, $result$$Register, StrIntrinsicNode::UU);
16968   %}
16969   ins_pipe(pipe_class_memory);
16970 %}
16971 
16972 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16973        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16974        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16975 %{
16976   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16977   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16978   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16979          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16980   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16981 
16982   ins_encode %{
16983     __ string_indexof($str1$$Register, $str2$$Register,
16984                       $cnt1$$Register, $cnt2$$Register,
16985                       $tmp1$$Register, $tmp2$$Register,
16986                       $tmp3$$Register, $tmp4$$Register,
16987                       $tmp5$$Register, $tmp6$$Register,
16988                       -1, $result$$Register, StrIntrinsicNode::LL);
16989   %}
16990   ins_pipe(pipe_class_memory);
16991 %}
16992 
16993 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16994        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16995        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16996 %{
16997   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16998   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16999   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17000          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17001   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17002 
17003   ins_encode %{
17004     __ string_indexof($str1$$Register, $str2$$Register,
17005                       $cnt1$$Register, $cnt2$$Register,
17006                       $tmp1$$Register, $tmp2$$Register,
17007                       $tmp3$$Register, $tmp4$$Register,
17008                       $tmp5$$Register, $tmp6$$Register,
17009                       -1, $result$$Register, StrIntrinsicNode::UL);
17010   %}
17011   ins_pipe(pipe_class_memory);
17012 %}
17013 
17014 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17015                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17016                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17017 %{
17018   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17019   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17020   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17021          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17022   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17023 
17024   ins_encode %{
17025     int icnt2 = (int)$int_cnt2$$constant;
17026     __ string_indexof($str1$$Register, $str2$$Register,
17027                       $cnt1$$Register, zr,
17028                       $tmp1$$Register, $tmp2$$Register,
17029                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17030                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17031   %}
17032   ins_pipe(pipe_class_memory);
17033 %}
17034 
17035 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17036                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17037                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17038 %{
17039   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17040   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17041   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17042          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17043   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17044 
17045   ins_encode %{
17046     int icnt2 = (int)$int_cnt2$$constant;
17047     __ string_indexof($str1$$Register, $str2$$Register,
17048                       $cnt1$$Register, zr,
17049                       $tmp1$$Register, $tmp2$$Register,
17050                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17051                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17052   %}
17053   ins_pipe(pipe_class_memory);
17054 %}
17055 
17056 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17057                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17058                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17059 %{
17060   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17061   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17062   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17063          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17064   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17065 
17066   ins_encode %{
17067     int icnt2 = (int)$int_cnt2$$constant;
17068     __ string_indexof($str1$$Register, $str2$$Register,
17069                       $cnt1$$Register, zr,
17070                       $tmp1$$Register, $tmp2$$Register,
17071                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17072                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17073   %}
17074   ins_pipe(pipe_class_memory);
17075 %}
17076 
17077 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17078                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17079                              iRegINoSp tmp3, rFlagsReg cr)
17080 %{
17081   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17082   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17083   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17084          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17085 
17086   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17087 
17088   ins_encode %{
17089     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17090                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17091                            $tmp3$$Register);
17092   %}
17093   ins_pipe(pipe_class_memory);
17094 %}
17095 
17096 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17097                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17098                               iRegINoSp tmp3, rFlagsReg cr)
17099 %{
17100   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17101   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17102   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17103          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17104 
17105   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17106 
17107   ins_encode %{
17108     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17109                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17110                             $tmp3$$Register);
17111   %}
17112   ins_pipe(pipe_class_memory);
17113 %}
17114 
17115 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17116                         iRegI_R0 result, rFlagsReg cr)
17117 %{
17118   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17119   match(Set result (StrEquals (Binary str1 str2) cnt));
17120   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17121 
17122   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17123   ins_encode %{
17124     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17125     __ string_equals($str1$$Register, $str2$$Register,
17126                      $result$$Register, $cnt$$Register, 1);
17127   %}
17128   ins_pipe(pipe_class_memory);
17129 %}
17130 
17131 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17132                         iRegI_R0 result, rFlagsReg cr)
17133 %{
17134   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17135   match(Set result (StrEquals (Binary str1 str2) cnt));
17136   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17137 
17138   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17139   ins_encode %{
17140     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17141     __ string_equals($str1$$Register, $str2$$Register,
17142                      $result$$Register, $cnt$$Register, 2);
17143   %}
17144   ins_pipe(pipe_class_memory);
17145 %}
17146 
17147 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17148                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17149                        iRegP_R10 tmp, rFlagsReg cr)
17150 %{
17151   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17152   match(Set result (AryEq ary1 ary2));
17153   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17154 
17155   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17156   ins_encode %{
17157     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17158                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17159                                    $result$$Register, $tmp$$Register, 1);
17160     if (tpc == NULL) {
17161       ciEnv::current()->record_failure("CodeCache is full");
17162       return;
17163     }
17164   %}
17165   ins_pipe(pipe_class_memory);
17166 %}
17167 
17168 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17169                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17170                        iRegP_R10 tmp, rFlagsReg cr)
17171 %{
17172   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17173   match(Set result (AryEq ary1 ary2));
17174   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17175 
17176   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17177   ins_encode %{
17178     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17179                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17180                                    $result$$Register, $tmp$$Register, 2);
17181     if (tpc == NULL) {
17182       ciEnv::current()->record_failure("CodeCache is full");
17183       return;
17184     }
17185   %}
17186   ins_pipe(pipe_class_memory);
17187 %}
17188 
17189 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17190 %{
17191   match(Set result (CountPositives ary1 len));
17192   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17193   format %{ "count positives byte[] $ary1,$len -> $result" %}
17194   ins_encode %{
17195     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17196     if (tpc == NULL) {
17197       ciEnv::current()->record_failure("CodeCache is full");
17198       return;
17199     }
17200   %}
17201   ins_pipe( pipe_slow );
17202 %}
17203 
17204 // fast char[] to byte[] compression
17205 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17206                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17207                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17208                          iRegI_R0 result, rFlagsReg cr)
17209 %{
17210   match(Set result (StrCompressedCopy src (Binary dst len)));
17211   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17212          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17213 
17214   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
17215   ins_encode %{
17216     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17217                            $result$$Register,
17218                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17219                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17220   %}
17221   ins_pipe(pipe_slow);
17222 %}
17223 
17224 // fast byte[] to char[] inflation
17225 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17226                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17227 %{
17228   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17229   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17230 
17231   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17232   ins_encode %{
17233     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17234                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17235                                         $tmp3$$FloatRegister, $tmp4$$Register);
17236     if (tpc == NULL) {
17237       ciEnv::current()->record_failure("CodeCache is full");
17238       return;
17239     }
17240   %}
17241   ins_pipe(pipe_class_memory);
17242 %}
17243 
17244 // encode char[] to byte[] in ISO_8859_1
17245 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17246                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17247                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17248                           iRegI_R0 result, rFlagsReg cr)
17249 %{
17250   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17251   match(Set result (EncodeISOArray src (Binary dst len)));
17252   effect(USE_KILL src, USE_KILL dst, USE len,
17253          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17254 
17255   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17256   ins_encode %{
17257     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17258                         $result$$Register, false,
17259                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17260                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17261   %}
17262   ins_pipe(pipe_class_memory);
17263 %}
17264 
17265 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17266                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17267                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17268                             iRegI_R0 result, rFlagsReg cr)
17269 %{
17270   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17271   match(Set result (EncodeISOArray src (Binary dst len)));
17272   effect(USE_KILL src, USE_KILL dst, USE len,
17273          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17274 
17275   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17276   ins_encode %{
17277     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17278                         $result$$Register, true,
17279                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17280                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17281   %}
17282   ins_pipe(pipe_class_memory);
17283 %}
17284 
17285 // ============================================================================
17286 // This name is KNOWN by the ADLC and cannot be changed.
17287 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17288 // for this guy.
17289 instruct tlsLoadP(thread_RegP dst)
17290 %{
17291   match(Set dst (ThreadLocal));
17292 
17293   ins_cost(0);
17294 
17295   format %{ " -- \t// $dst=Thread::current(), empty" %}
17296 
17297   size(0);
17298 
17299   ins_encode( /*empty*/ );
17300 
17301   ins_pipe(pipe_class_empty);
17302 %}
17303 
17304 //----------PEEPHOLE RULES-----------------------------------------------------
17305 // These must follow all instruction definitions as they use the names
17306 // defined in the instructions definitions.
17307 //
17308 // peepmatch ( root_instr_name [preceding_instruction]* );
17309 //
17310 // peepconstraint %{
17311 // (instruction_number.operand_name relational_op instruction_number.operand_name
17312 //  [, ...] );
17313 // // instruction numbers are zero-based using left to right order in peepmatch
17314 //
17315 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17316 // // provide an instruction_number.operand_name for each operand that appears
17317 // // in the replacement instruction's match rule
17318 //
17319 // ---------VM FLAGS---------------------------------------------------------
17320 //
17321 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17322 //
17323 // Each peephole rule is given an identifying number starting with zero and
17324 // increasing by one in the order seen by the parser.  An individual peephole
17325 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17326 // on the command-line.
17327 //
17328 // ---------CURRENT LIMITATIONS----------------------------------------------
17329 //
17330 // Only match adjacent instructions in same basic block
17331 // Only equality constraints
17332 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17333 // Only one replacement instruction
17334 //
17335 // ---------EXAMPLE----------------------------------------------------------
17336 //
17337 // // pertinent parts of existing instructions in architecture description
17338 // instruct movI(iRegINoSp dst, iRegI src)
17339 // %{
17340 //   match(Set dst (CopyI src));
17341 // %}
17342 //
17343 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17344 // %{
17345 //   match(Set dst (AddI dst src));
17346 //   effect(KILL cr);
17347 // %}
17348 //
17349 // // Change (inc mov) to lea
17350 // peephole %{
17351 //   // increment preceeded by register-register move
17352 //   peepmatch ( incI_iReg movI );
17353 //   // require that the destination register of the increment
17354 //   // match the destination register of the move
17355 //   peepconstraint ( 0.dst == 1.dst );
17356 //   // construct a replacement instruction that sets
17357 //   // the destination to ( move's source register + one )
17358 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17359 // %}
17360 //
17361 
17362 // Implementation no longer uses movX instructions since
17363 // machine-independent system no longer uses CopyX nodes.
17364 //
17365 // peephole
17366 // %{
17367 //   peepmatch (incI_iReg movI);
17368 //   peepconstraint (0.dst == 1.dst);
17369 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17370 // %}
17371 
17372 // peephole
17373 // %{
17374 //   peepmatch (decI_iReg movI);
17375 //   peepconstraint (0.dst == 1.dst);
17376 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17377 // %}
17378 
17379 // peephole
17380 // %{
17381 //   peepmatch (addI_iReg_imm movI);
17382 //   peepconstraint (0.dst == 1.dst);
17383 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17384 // %}
17385 
17386 // peephole
17387 // %{
17388 //   peepmatch (incL_iReg movL);
17389 //   peepconstraint (0.dst == 1.dst);
17390 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17391 // %}
17392 
17393 // peephole
17394 // %{
17395 //   peepmatch (decL_iReg movL);
17396 //   peepconstraint (0.dst == 1.dst);
17397 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17398 // %}
17399 
17400 // peephole
17401 // %{
17402 //   peepmatch (addL_iReg_imm movL);
17403 //   peepconstraint (0.dst == 1.dst);
17404 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17405 // %}
17406 
17407 // peephole
17408 // %{
17409 //   peepmatch (addP_iReg_imm movP);
17410 //   peepconstraint (0.dst == 1.dst);
17411 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17412 // %}
17413 
17414 // // Change load of spilled value to only a spill
17415 // instruct storeI(memory mem, iRegI src)
17416 // %{
17417 //   match(Set mem (StoreI mem src));
17418 // %}
17419 //
17420 // instruct loadI(iRegINoSp dst, memory mem)
17421 // %{
17422 //   match(Set dst (LoadI mem));
17423 // %}
17424 //
17425 
17426 //----------SMARTSPILL RULES---------------------------------------------------
17427 // These must follow all instruction definitions as they use the names
17428 // defined in the instructions definitions.
17429 
17430 // Local Variables:
17431 // mode: c++
17432 // End: