1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly accessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_codestub_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298   bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // Assert that the given node is not a variable shift.
 1315   bool assert_not_var_shift(const Node* n);
 1316 
 1317   // predicate controlling addressing modes
 1318   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1319 %}
 1320 
 1321 source %{
 1322 
 1323   // Derived RegMask with conditionally allocatable registers
 1324 
 1325   void PhaseOutput::pd_perform_mach_node_analysis() {
 1326   }
 1327 
 1328   int MachNode::pd_alignment_required() const {
 1329     return 1;
 1330   }
 1331 
 1332   int MachNode::compute_padding(int current_offset) const {
 1333     return 0;
 1334   }
 1335 
 1336   RegMask _ANY_REG32_mask;
 1337   RegMask _ANY_REG_mask;
 1338   RegMask _PTR_REG_mask;
 1339   RegMask _NO_SPECIAL_REG32_mask;
 1340   RegMask _NO_SPECIAL_REG_mask;
 1341   RegMask _NO_SPECIAL_PTR_REG_mask;
 1342 
 1343   void reg_mask_init() {
 1344     // We derive below RegMask(s) from the ones which are auto-generated from
 1345     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1346     // registers conditionally reserved.
 1347 
 1348     _ANY_REG32_mask = _ALL_REG32_mask;
 1349     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1350 
 1351     _ANY_REG_mask = _ALL_REG_mask;
 1352 
 1353     _PTR_REG_mask = _ALL_REG_mask;
 1354 
 1355     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1356     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1357 
 1358     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1362     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1363 
 1364     // r27 is not allocatable when compressed oops is on and heapbase is not
 1365     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1366     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1367       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1368       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1369       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1370     }
 1371 
 1372     // r29 is not allocatable when PreserveFramePointer is on
 1373     if (PreserveFramePointer) {
 1374       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1375       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1376       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1377     }
 1378   }
 1379 
 1380   // Optimizaton of volatile gets and puts
 1381   // -------------------------------------
 1382   //
 1383   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1384   // use to implement volatile reads and writes. For a volatile read
 1385   // we simply need
 1386   //
 1387   //   ldar<x>
 1388   //
 1389   // and for a volatile write we need
 1390   //
 1391   //   stlr<x>
 1392   //
 1393   // Alternatively, we can implement them by pairing a normal
 1394   // load/store with a memory barrier. For a volatile read we need
 1395   //
 1396   //   ldr<x>
 1397   //   dmb ishld
 1398   //
 1399   // for a volatile write
 1400   //
 1401   //   dmb ish
 1402   //   str<x>
 1403   //   dmb ish
 1404   //
 1405   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1406   // sequences. These are normally translated to an instruction
 1407   // sequence like the following
 1408   //
 1409   //   dmb      ish
 1410   // retry:
 1411   //   ldxr<x>   rval raddr
 1412   //   cmp       rval rold
 1413   //   b.ne done
 1414   //   stlxr<x>  rval, rnew, rold
 1415   //   cbnz      rval retry
 1416   // done:
 1417   //   cset      r0, eq
 1418   //   dmb ishld
 1419   //
 1420   // Note that the exclusive store is already using an stlxr
 1421   // instruction. That is required to ensure visibility to other
 1422   // threads of the exclusive write (assuming it succeeds) before that
 1423   // of any subsequent writes.
 1424   //
 1425   // The following instruction sequence is an improvement on the above
 1426   //
 1427   // retry:
 1428   //   ldaxr<x>  rval raddr
 1429   //   cmp       rval rold
 1430   //   b.ne done
 1431   //   stlxr<x>  rval, rnew, rold
 1432   //   cbnz      rval retry
 1433   // done:
 1434   //   cset      r0, eq
 1435   //
 1436   // We don't need the leading dmb ish since the stlxr guarantees
 1437   // visibility of prior writes in the case that the swap is
 1438   // successful. Crucially we don't have to worry about the case where
 1439   // the swap is not successful since no valid program should be
 1440   // relying on visibility of prior changes by the attempting thread
 1441   // in the case where the CAS fails.
 1442   //
 1443   // Similarly, we don't need the trailing dmb ishld if we substitute
 1444   // an ldaxr instruction since that will provide all the guarantees we
 1445   // require regarding observation of changes made by other threads
 1446   // before any change to the CAS address observed by the load.
 1447   //
 1448   // In order to generate the desired instruction sequence we need to
 1449   // be able to identify specific 'signature' ideal graph node
 1450   // sequences which i) occur as a translation of a volatile reads or
 1451   // writes or CAS operations and ii) do not occur through any other
 1452   // translation or graph transformation. We can then provide
 1453   // alternative aldc matching rules which translate these node
 1454   // sequences to the desired machine code sequences. Selection of the
 1455   // alternative rules can be implemented by predicates which identify
 1456   // the relevant node sequences.
 1457   //
 1458   // The ideal graph generator translates a volatile read to the node
 1459   // sequence
 1460   //
 1461   //   LoadX[mo_acquire]
 1462   //   MemBarAcquire
 1463   //
 1464   // As a special case when using the compressed oops optimization we
 1465   // may also see this variant
 1466   //
 1467   //   LoadN[mo_acquire]
 1468   //   DecodeN
 1469   //   MemBarAcquire
 1470   //
 1471   // A volatile write is translated to the node sequence
 1472   //
 1473   //   MemBarRelease
 1474   //   StoreX[mo_release] {CardMark}-optional
 1475   //   MemBarVolatile
 1476   //
 1477   // n.b. the above node patterns are generated with a strict
 1478   // 'signature' configuration of input and output dependencies (see
 1479   // the predicates below for exact details). The card mark may be as
 1480   // simple as a few extra nodes or, in a few GC configurations, may
 1481   // include more complex control flow between the leading and
 1482   // trailing memory barriers. However, whatever the card mark
 1483   // configuration these signatures are unique to translated volatile
 1484   // reads/stores -- they will not appear as a result of any other
 1485   // bytecode translation or inlining nor as a consequence of
 1486   // optimizing transforms.
 1487   //
 1488   // We also want to catch inlined unsafe volatile gets and puts and
 1489   // be able to implement them using either ldar<x>/stlr<x> or some
 1490   // combination of ldr<x>/stlr<x> and dmb instructions.
 1491   //
 1492   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1493   // normal volatile put node sequence containing an extra cpuorder
 1494   // membar
 1495   //
 1496   //   MemBarRelease
 1497   //   MemBarCPUOrder
 1498   //   StoreX[mo_release] {CardMark}-optional
 1499   //   MemBarCPUOrder
 1500   //   MemBarVolatile
 1501   //
 1502   // n.b. as an aside, a cpuorder membar is not itself subject to
 1503   // matching and translation by adlc rules.  However, the rule
 1504   // predicates need to detect its presence in order to correctly
 1505   // select the desired adlc rules.
 1506   //
 1507   // Inlined unsafe volatile gets manifest as a slightly different
 1508   // node sequence to a normal volatile get because of the
 1509   // introduction of some CPUOrder memory barriers to bracket the
 1510   // Load. However, but the same basic skeleton of a LoadX feeding a
 1511   // MemBarAcquire, possibly through an optional DecodeN, is still
 1512   // present
 1513   //
 1514   //   MemBarCPUOrder
 1515   //        ||       \\
 1516   //   MemBarCPUOrder LoadX[mo_acquire]
 1517   //        ||            |
 1518   //        ||       {DecodeN} optional
 1519   //        ||       /
 1520   //     MemBarAcquire
 1521   //
 1522   // In this case the acquire membar does not directly depend on the
 1523   // load. However, we can be sure that the load is generated from an
 1524   // inlined unsafe volatile get if we see it dependent on this unique
 1525   // sequence of membar nodes. Similarly, given an acquire membar we
 1526   // can know that it was added because of an inlined unsafe volatile
 1527   // get if it is fed and feeds a cpuorder membar and if its feed
 1528   // membar also feeds an acquiring load.
 1529   //
 1530   // Finally an inlined (Unsafe) CAS operation is translated to the
 1531   // following ideal graph
 1532   //
 1533   //   MemBarRelease
 1534   //   MemBarCPUOrder
 1535   //   CompareAndSwapX {CardMark}-optional
 1536   //   MemBarCPUOrder
 1537   //   MemBarAcquire
 1538   //
 1539   // So, where we can identify these volatile read and write
 1540   // signatures we can choose to plant either of the above two code
 1541   // sequences. For a volatile read we can simply plant a normal
 1542   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1543   // also choose to inhibit translation of the MemBarAcquire and
 1544   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1545   //
 1546   // When we recognise a volatile store signature we can choose to
 1547   // plant at a dmb ish as a translation for the MemBarRelease, a
 1548   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1549   // Alternatively, we can inhibit translation of the MemBarRelease
 1550   // and MemBarVolatile and instead plant a simple stlr<x>
 1551   // instruction.
 1552   //
 1553   // when we recognise a CAS signature we can choose to plant a dmb
 1554   // ish as a translation for the MemBarRelease, the conventional
 1555   // macro-instruction sequence for the CompareAndSwap node (which
 1556   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1557   // Alternatively, we can elide generation of the dmb instructions
 1558   // and plant the alternative CompareAndSwap macro-instruction
 1559   // sequence (which uses ldaxr<x>).
 1560   //
 1561   // Of course, the above only applies when we see these signature
 1562   // configurations. We still want to plant dmb instructions in any
 1563   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1564   // MemBarVolatile. For example, at the end of a constructor which
 1565   // writes final/volatile fields we will see a MemBarRelease
 1566   // instruction and this needs a 'dmb ish' lest we risk the
 1567   // constructed object being visible without making the
 1568   // final/volatile field writes visible.
 1569   //
 1570   // n.b. the translation rules below which rely on detection of the
 1571   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1572   // If we see anything other than the signature configurations we
 1573   // always just translate the loads and stores to ldr<x> and str<x>
 1574   // and translate acquire, release and volatile membars to the
 1575   // relevant dmb instructions.
 1576   //
 1577 
 1578   // is_CAS(int opcode, bool maybe_volatile)
 1579   //
 1580   // return true if opcode is one of the possible CompareAndSwapX
 1581   // values otherwise false.
 1582 
 1583   bool is_CAS(int opcode, bool maybe_volatile)
 1584   {
 1585     switch(opcode) {
 1586       // We handle these
 1587     case Op_CompareAndSwapI:
 1588     case Op_CompareAndSwapL:
 1589     case Op_CompareAndSwapP:
 1590     case Op_CompareAndSwapN:
 1591     case Op_ShenandoahCompareAndSwapP:
 1592     case Op_ShenandoahCompareAndSwapN:
 1593     case Op_CompareAndSwapB:
 1594     case Op_CompareAndSwapS:
 1595     case Op_GetAndSetI:
 1596     case Op_GetAndSetL:
 1597     case Op_GetAndSetP:
 1598     case Op_GetAndSetN:
 1599     case Op_GetAndAddI:
 1600     case Op_GetAndAddL:
 1601       return true;
 1602     case Op_CompareAndExchangeI:
 1603     case Op_CompareAndExchangeN:
 1604     case Op_CompareAndExchangeB:
 1605     case Op_CompareAndExchangeS:
 1606     case Op_CompareAndExchangeL:
 1607     case Op_CompareAndExchangeP:
 1608     case Op_WeakCompareAndSwapB:
 1609     case Op_WeakCompareAndSwapS:
 1610     case Op_WeakCompareAndSwapI:
 1611     case Op_WeakCompareAndSwapL:
 1612     case Op_WeakCompareAndSwapP:
 1613     case Op_WeakCompareAndSwapN:
 1614     case Op_ShenandoahWeakCompareAndSwapP:
 1615     case Op_ShenandoahWeakCompareAndSwapN:
 1616     case Op_ShenandoahCompareAndExchangeP:
 1617     case Op_ShenandoahCompareAndExchangeN:
 1618       return maybe_volatile;
 1619     default:
 1620       return false;
 1621     }
 1622   }
 1623 
 1624   // helper to determine the maximum number of Phi nodes we may need to
 1625   // traverse when searching from a card mark membar for the merge mem
 1626   // feeding a trailing membar or vice versa
 1627 
 1628 // predicates controlling emit of ldr<x>/ldar<x>
 1629 
 1630 bool unnecessary_acquire(const Node *barrier)
 1631 {
 1632   assert(barrier->is_MemBar(), "expecting a membar");
 1633 
 1634   MemBarNode* mb = barrier->as_MemBar();
 1635 
 1636   if (mb->trailing_load()) {
 1637     return true;
 1638   }
 1639 
 1640   if (mb->trailing_load_store()) {
 1641     Node* load_store = mb->in(MemBarNode::Precedent);
 1642     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1643     return is_CAS(load_store->Opcode(), true);
 1644   }
 1645 
 1646   return false;
 1647 }
 1648 
 1649 bool needs_acquiring_load(const Node *n)
 1650 {
 1651   assert(n->is_Load(), "expecting a load");
 1652   LoadNode *ld = n->as_Load();
 1653   return ld->is_acquire();
 1654 }
 1655 
 1656 bool unnecessary_release(const Node *n)
 1657 {
 1658   assert((n->is_MemBar() &&
 1659           n->Opcode() == Op_MemBarRelease),
 1660          "expecting a release membar");
 1661 
 1662   MemBarNode *barrier = n->as_MemBar();
 1663   if (!barrier->leading()) {
 1664     return false;
 1665   } else {
 1666     Node* trailing = barrier->trailing_membar();
 1667     MemBarNode* trailing_mb = trailing->as_MemBar();
 1668     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1669     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1670 
 1671     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1672     if (mem->is_Store()) {
 1673       assert(mem->as_Store()->is_release(), "");
 1674       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1675       return true;
 1676     } else {
 1677       assert(mem->is_LoadStore(), "");
 1678       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1679       return is_CAS(mem->Opcode(), true);
 1680     }
 1681   }
 1682   return false;
 1683 }
 1684 
 1685 bool unnecessary_volatile(const Node *n)
 1686 {
 1687   // assert n->is_MemBar();
 1688   MemBarNode *mbvol = n->as_MemBar();
 1689 
 1690   bool release = mbvol->trailing_store();
 1691   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1692 #ifdef ASSERT
 1693   if (release) {
 1694     Node* leading = mbvol->leading_membar();
 1695     assert(leading->Opcode() == Op_MemBarRelease, "");
 1696     assert(leading->as_MemBar()->leading_store(), "");
 1697     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1698   }
 1699 #endif
 1700 
 1701   return release;
 1702 }
 1703 
 1704 // predicates controlling emit of str<x>/stlr<x>
 1705 
 1706 bool needs_releasing_store(const Node *n)
 1707 {
 1708   // assert n->is_Store();
 1709   StoreNode *st = n->as_Store();
 1710   return st->trailing_membar() != NULL;
 1711 }
 1712 
 1713 // predicate controlling translation of CAS
 1714 //
 1715 // returns true if CAS needs to use an acquiring load otherwise false
 1716 
 1717 bool needs_acquiring_load_exclusive(const Node *n)
 1718 {
 1719   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1720   LoadStoreNode* ldst = n->as_LoadStore();
 1721   if (is_CAS(n->Opcode(), false)) {
 1722     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1723   } else {
 1724     return ldst->trailing_membar() != NULL;
 1725   }
 1726 
 1727   // so we can just return true here
 1728   return true;
 1729 }
 1730 
 1731 // Assert that the given node is not a variable shift.
 1732 bool assert_not_var_shift(const Node* n) {
 1733   assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
 1734   return true;
 1735 }
 1736 
 1737 #define __ _masm.
 1738 
 1739 // advance declarations for helper functions to convert register
 1740 // indices to register objects
 1741 
 1742 // the ad file has to provide implementations of certain methods
 1743 // expected by the generic code
 1744 //
 1745 // REQUIRED FUNCTIONALITY
 1746 
 1747 //=============================================================================
 1748 
 1749 // !!!!! Special hack to get all types of calls to specify the byte offset
 1750 //       from the start of the call to the point where the return address
 1751 //       will point.
 1752 
 1753 int MachCallStaticJavaNode::ret_addr_offset()
 1754 {
 1755   // call should be a simple bl
 1756   int off = 4;
 1757   return off;
 1758 }
 1759 
 1760 int MachCallDynamicJavaNode::ret_addr_offset()
 1761 {
 1762   return 16; // movz, movk, movk, bl
 1763 }
 1764 
 1765 int MachCallRuntimeNode::ret_addr_offset() {
 1766   // for generated stubs the call will be
 1767   //   bl(addr)
 1768   // or with far branches
 1769   //   bl(trampoline_stub)
 1770   // for real runtime callouts it will be six instructions
 1771   // see aarch64_enc_java_to_runtime
 1772   //   adr(rscratch2, retaddr)
 1773   //   lea(rscratch1, RuntimeAddress(addr)
 1774   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1775   //   blr(rscratch1)
 1776   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1777   if (cb) {
 1778     return 1 * NativeInstruction::instruction_size;
 1779   } else {
 1780     return 6 * NativeInstruction::instruction_size;
 1781   }
 1782 }
 1783 
 1784 int MachCallNativeNode::ret_addr_offset() {
 1785   // This is implemented using aarch64_enc_java_to_runtime as above.
 1786   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1787   if (cb) {
 1788     return 1 * NativeInstruction::instruction_size;
 1789   } else {
 1790     return 6 * NativeInstruction::instruction_size;
 1791   }
 1792 }
 1793 
 1794 //=============================================================================
 1795 
 1796 #ifndef PRODUCT
 1797 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1798   st->print("BREAKPOINT");
 1799 }
 1800 #endif
 1801 
 1802 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1803   C2_MacroAssembler _masm(&cbuf);
 1804   __ brk(0);
 1805 }
 1806 
 1807 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1808   return MachNode::size(ra_);
 1809 }
 1810 
 1811 //=============================================================================
 1812 
 1813 #ifndef PRODUCT
 1814   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1815     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1816   }
 1817 #endif
 1818 
 1819   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1820     C2_MacroAssembler _masm(&cbuf);
 1821     for (int i = 0; i < _count; i++) {
 1822       __ nop();
 1823     }
 1824   }
 1825 
 1826   uint MachNopNode::size(PhaseRegAlloc*) const {
 1827     return _count * NativeInstruction::instruction_size;
 1828   }
 1829 
 1830 //=============================================================================
 1831 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1832 
 1833 int ConstantTable::calculate_table_base_offset() const {
 1834   return 0;  // absolute addressing, no offset
 1835 }
 1836 
 1837 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1838 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1839   ShouldNotReachHere();
 1840 }
 1841 
 1842 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1843   // Empty encoding
 1844 }
 1845 
 1846 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1847   return 0;
 1848 }
 1849 
 1850 #ifndef PRODUCT
 1851 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1852   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1853 }
 1854 #endif
 1855 
 1856 #ifndef PRODUCT
 1857 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1858   Compile* C = ra_->C;
 1859 
 1860   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1861 
 1862   if (C->output()->need_stack_bang(framesize))
 1863     st->print("# stack bang size=%d\n\t", framesize);
 1864 
 1865   if (VM_Version::use_rop_protection()) {
 1866     st->print("ldr zr, [lr]\n\t");
 1867     st->print("pacia  lr, rfp\n\t");
 1868   }
 1869   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1870     st->print("sub  sp, sp, #%d\n\t", framesize);
 1871     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1872     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1873   } else {
 1874     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1875     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1876     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1877     st->print("sub  sp, sp, rscratch1");
 1878   }
 1879   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1880     st->print("\n\t");
 1881     st->print("ldr  rscratch1, [guard]\n\t");
 1882     st->print("dmb ishld\n\t");
 1883     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1884     st->print("cmp  rscratch1, rscratch2\n\t");
 1885     st->print("b.eq skip");
 1886     st->print("\n\t");
 1887     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1888     st->print("b skip\n\t");
 1889     st->print("guard: int\n\t");
 1890     st->print("\n\t");
 1891     st->print("skip:\n\t");
 1892   }
 1893 }
 1894 #endif
 1895 
 1896 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1897   Compile* C = ra_->C;
 1898   C2_MacroAssembler _masm(&cbuf);
 1899 
 1900   // n.b. frame size includes space for return pc and rfp
 1901   const int framesize = C->output()->frame_size_in_bytes();
 1902 
 1903   // insert a nop at the start of the prolog so we can patch in a
 1904   // branch if we need to invalidate the method later
 1905   __ nop();
 1906 
 1907   if (C->clinit_barrier_on_entry()) {
 1908     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1909 
 1910     Label L_skip_barrier;
 1911 
 1912     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1913     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1914     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1915     __ bind(L_skip_barrier);
 1916   }
 1917 
 1918   if (C->max_vector_size() > 0) {
 1919     __ reinitialize_ptrue();
 1920   }
 1921 
 1922   int bangsize = C->output()->bang_size_in_bytes();
 1923   if (C->output()->need_stack_bang(bangsize))
 1924     __ generate_stack_overflow_check(bangsize);
 1925 
 1926   __ build_frame(framesize);
 1927 
 1928   if (C->stub_function() == NULL) {
 1929     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1930     bs->nmethod_entry_barrier(&_masm);
 1931   }
 1932 
 1933   if (VerifyStackAtCalls) {
 1934     Unimplemented();
 1935   }
 1936 
 1937   C->output()->set_frame_complete(cbuf.insts_size());
 1938 
 1939   if (C->has_mach_constant_base_node()) {
 1940     // NOTE: We set the table base offset here because users might be
 1941     // emitted before MachConstantBaseNode.
 1942     ConstantTable& constant_table = C->output()->constant_table();
 1943     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1944   }
 1945 }
 1946 
 1947 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1948 {
 1949   return MachNode::size(ra_); // too many variables; just compute it
 1950                               // the hard way
 1951 }
 1952 
 1953 int MachPrologNode::reloc() const
 1954 {
 1955   return 0;
 1956 }
 1957 
 1958 //=============================================================================
 1959 
 1960 #ifndef PRODUCT
 1961 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1962   Compile* C = ra_->C;
 1963   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1964 
 1965   st->print("# pop frame %d\n\t",framesize);
 1966 
 1967   if (framesize == 0) {
 1968     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1969   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1970     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1971     st->print("add  sp, sp, #%d\n\t", framesize);
 1972   } else {
 1973     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1974     st->print("add  sp, sp, rscratch1\n\t");
 1975     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1976   }
 1977   if (VM_Version::use_rop_protection()) {
 1978     st->print("autia lr, rfp\n\t");
 1979     st->print("ldr zr, [lr]\n\t");
 1980   }
 1981 
 1982   if (do_polling() && C->is_method_compilation()) {
 1983     st->print("# test polling word\n\t");
 1984     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1985     st->print("cmp  sp, rscratch1\n\t");
 1986     st->print("bhi #slow_path");
 1987   }
 1988 }
 1989 #endif
 1990 
 1991 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1992   Compile* C = ra_->C;
 1993   C2_MacroAssembler _masm(&cbuf);
 1994   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1995 
 1996   __ remove_frame(framesize);
 1997 
 1998   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1999     __ reserved_stack_check();
 2000   }
 2001 
 2002   if (do_polling() && C->is_method_compilation()) {
 2003     Label dummy_label;
 2004     Label* code_stub = &dummy_label;
 2005     if (!C->output()->in_scratch_emit_size()) {
 2006       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 2007     }
 2008     __ relocate(relocInfo::poll_return_type);
 2009     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 2010   }
 2011 }
 2012 
 2013 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 2014   // Variable size. Determine dynamically.
 2015   return MachNode::size(ra_);
 2016 }
 2017 
 2018 int MachEpilogNode::reloc() const {
 2019   // Return number of relocatable values contained in this instruction.
 2020   return 1; // 1 for polling page.
 2021 }
 2022 
 2023 const Pipeline * MachEpilogNode::pipeline() const {
 2024   return MachNode::pipeline_class();
 2025 }
 2026 
 2027 //=============================================================================
 2028 
 2029 // Figure out which register class each belongs in: rc_int, rc_float or
 2030 // rc_stack.
 2031 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2032 
 2033 static enum RC rc_class(OptoReg::Name reg) {
 2034 
 2035   if (reg == OptoReg::Bad) {
 2036     return rc_bad;
 2037   }
 2038 
 2039   // we have 32 int registers * 2 halves
 2040   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2041 
 2042   if (reg < slots_of_int_registers) {
 2043     return rc_int;
 2044   }
 2045 
 2046   // we have 32 float register * 8 halves
 2047   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2048   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2049     return rc_float;
 2050   }
 2051 
 2052   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2053   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2054     return rc_predicate;
 2055   }
 2056 
 2057   // Between predicate regs & stack is the flags.
 2058   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2059 
 2060   return rc_stack;
 2061 }
 2062 
 2063 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2064   Compile* C = ra_->C;
 2065 
 2066   // Get registers to move.
 2067   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2068   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2069   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2070   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2071 
 2072   enum RC src_hi_rc = rc_class(src_hi);
 2073   enum RC src_lo_rc = rc_class(src_lo);
 2074   enum RC dst_hi_rc = rc_class(dst_hi);
 2075   enum RC dst_lo_rc = rc_class(dst_lo);
 2076 
 2077   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2078 
 2079   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 2080     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2081            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2082            "expected aligned-adjacent pairs");
 2083   }
 2084 
 2085   if (src_lo == dst_lo && src_hi == dst_hi) {
 2086     return 0;            // Self copy, no move.
 2087   }
 2088 
 2089   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2090               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2091   int src_offset = ra_->reg2offset(src_lo);
 2092   int dst_offset = ra_->reg2offset(dst_lo);
 2093 
 2094   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2095     uint ireg = ideal_reg();
 2096     if (ireg == Op_VecA && cbuf) {
 2097       C2_MacroAssembler _masm(cbuf);
 2098       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2099       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2100         // stack->stack
 2101         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2102                                                 sve_vector_reg_size_in_bytes);
 2103       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2104         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2105                             sve_vector_reg_size_in_bytes);
 2106       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2107         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2108                               sve_vector_reg_size_in_bytes);
 2109       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2110         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2111                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2112                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2113       } else {
 2114         ShouldNotReachHere();
 2115       }
 2116     } else if (cbuf) {
 2117       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2118       C2_MacroAssembler _masm(cbuf);
 2119       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2120       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2121         // stack->stack
 2122         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2123         if (ireg == Op_VecD) {
 2124           __ unspill(rscratch1, true, src_offset);
 2125           __ spill(rscratch1, true, dst_offset);
 2126         } else {
 2127           __ spill_copy128(src_offset, dst_offset);
 2128         }
 2129       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2130         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2131                ireg == Op_VecD ? __ T8B : __ T16B,
 2132                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2133       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2134         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2135                  ireg == Op_VecD ? __ D : __ Q,
 2136                  ra_->reg2offset(dst_lo));
 2137       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2138         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2139                    ireg == Op_VecD ? __ D : __ Q,
 2140                    ra_->reg2offset(src_lo));
 2141       } else {
 2142         ShouldNotReachHere();
 2143       }
 2144     }
 2145   } else if (cbuf) {
 2146     C2_MacroAssembler _masm(cbuf);
 2147     switch (src_lo_rc) {
 2148     case rc_int:
 2149       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2150         if (is64) {
 2151             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2152                    as_Register(Matcher::_regEncode[src_lo]));
 2153         } else {
 2154             C2_MacroAssembler _masm(cbuf);
 2155             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2156                     as_Register(Matcher::_regEncode[src_lo]));
 2157         }
 2158       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2159         if (is64) {
 2160             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2161                      as_Register(Matcher::_regEncode[src_lo]));
 2162         } else {
 2163             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2164                      as_Register(Matcher::_regEncode[src_lo]));
 2165         }
 2166       } else {                    // gpr --> stack spill
 2167         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2168         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2169       }
 2170       break;
 2171     case rc_float:
 2172       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2173         if (is64) {
 2174             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2175                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2176         } else {
 2177             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2178                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2179         }
 2180       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2181         if (is64) {
 2182             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2183                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2184         } else {
 2185             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2186                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2187         }
 2188       } else {                    // fpr --> stack spill
 2189         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2190         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2191                  is64 ? __ D : __ S, dst_offset);
 2192       }
 2193       break;
 2194     case rc_stack:
 2195       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2196         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2197       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2198         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2199                    is64 ? __ D : __ S, src_offset);
 2200       } else if (dst_lo_rc == rc_predicate) {
 2201         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2202                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2203       } else {                    // stack --> stack copy
 2204         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2205         if (ideal_reg() == Op_RegVectMask) {
 2206           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2207                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2208         } else {
 2209           __ unspill(rscratch1, is64, src_offset);
 2210           __ spill(rscratch1, is64, dst_offset);
 2211         }
 2212       }
 2213       break;
 2214     case rc_predicate:
 2215       if (dst_lo_rc == rc_predicate) {
 2216         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2217       } else if (dst_lo_rc == rc_stack) {
 2218         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2219                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2220       } else {
 2221         assert(false, "bad src and dst rc_class combination.");
 2222         ShouldNotReachHere();
 2223       }
 2224       break;
 2225     default:
 2226       assert(false, "bad rc_class for spill");
 2227       ShouldNotReachHere();
 2228     }
 2229   }
 2230 
 2231   if (st) {
 2232     st->print("spill ");
 2233     if (src_lo_rc == rc_stack) {
 2234       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2235     } else {
 2236       st->print("%s -> ", Matcher::regName[src_lo]);
 2237     }
 2238     if (dst_lo_rc == rc_stack) {
 2239       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2240     } else {
 2241       st->print("%s", Matcher::regName[dst_lo]);
 2242     }
 2243     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2244       int vsize = 0;
 2245       switch (ideal_reg()) {
 2246       case Op_VecD:
 2247         vsize = 64;
 2248         break;
 2249       case Op_VecX:
 2250         vsize = 128;
 2251         break;
 2252       case Op_VecA:
 2253         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2254         break;
 2255       default:
 2256         assert(false, "bad register type for spill");
 2257         ShouldNotReachHere();
 2258       }
 2259       st->print("\t# vector spill size = %d", vsize);
 2260     } else if (ideal_reg() == Op_RegVectMask) {
 2261       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2262       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2263       st->print("\t# predicate spill size = %d", vsize);
 2264     } else {
 2265       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2266     }
 2267   }
 2268 
 2269   return 0;
 2270 
 2271 }
 2272 
 2273 #ifndef PRODUCT
 2274 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2275   if (!ra_)
 2276     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2277   else
 2278     implementation(NULL, ra_, false, st);
 2279 }
 2280 #endif
 2281 
 2282 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2283   implementation(&cbuf, ra_, false, NULL);
 2284 }
 2285 
 2286 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2287   return MachNode::size(ra_);
 2288 }
 2289 
 2290 //=============================================================================
 2291 
 2292 #ifndef PRODUCT
 2293 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2294   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2295   int reg = ra_->get_reg_first(this);
 2296   st->print("add %s, rsp, #%d]\t# box lock",
 2297             Matcher::regName[reg], offset);
 2298 }
 2299 #endif
 2300 
 2301 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2302   C2_MacroAssembler _masm(&cbuf);
 2303 
 2304   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2305   int reg    = ra_->get_encode(this);
 2306 
 2307   // This add will handle any 24-bit signed offset. 24 bits allows an
 2308   // 8 megabyte stack frame.
 2309   __ add(as_Register(reg), sp, offset);
 2310 }
 2311 
 2312 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2313   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2314   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2315 
 2316   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2317     return NativeInstruction::instruction_size;
 2318   } else {
 2319     return 2 * NativeInstruction::instruction_size;
 2320   }
 2321 }
 2322 
 2323 //=============================================================================
 2324 
 2325 #ifndef PRODUCT
 2326 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2327 {
 2328   st->print_cr("# MachUEPNode");
 2329   if (UseCompressedClassPointers) {
 2330     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2331     if (CompressedKlassPointers::shift() != 0) {
 2332       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2333     }
 2334   } else {
 2335    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2336   }
 2337   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2338   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2339 }
 2340 #endif
 2341 
 2342 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2343 {
 2344   // This is the unverified entry point.
 2345   C2_MacroAssembler _masm(&cbuf);
 2346 
 2347   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2348   Label skip;
 2349   // TODO
 2350   // can we avoid this skip and still use a reloc?
 2351   __ br(Assembler::EQ, skip);
 2352   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2353   __ bind(skip);
 2354 }
 2355 
 2356 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2357 {
 2358   return MachNode::size(ra_);
 2359 }
 2360 
 2361 // REQUIRED EMIT CODE
 2362 
 2363 //=============================================================================
 2364 
 2365 // Emit exception handler code.
 2366 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2367 {
 2368   // mov rscratch1 #exception_blob_entry_point
 2369   // br rscratch1
 2370   // Note that the code buffer's insts_mark is always relative to insts.
 2371   // That's why we must use the macroassembler to generate a handler.
 2372   C2_MacroAssembler _masm(&cbuf);
 2373   address base = __ start_a_stub(size_exception_handler());
 2374   if (base == NULL) {
 2375     ciEnv::current()->record_failure("CodeCache is full");
 2376     return 0;  // CodeBuffer::expand failed
 2377   }
 2378   int offset = __ offset();
 2379   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2380   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2381   __ end_a_stub();
 2382   return offset;
 2383 }
 2384 
 2385 // Emit deopt handler code.
 2386 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2387 {
 2388   // Note that the code buffer's insts_mark is always relative to insts.
 2389   // That's why we must use the macroassembler to generate a handler.
 2390   C2_MacroAssembler _masm(&cbuf);
 2391   address base = __ start_a_stub(size_deopt_handler());
 2392   if (base == NULL) {
 2393     ciEnv::current()->record_failure("CodeCache is full");
 2394     return 0;  // CodeBuffer::expand failed
 2395   }
 2396   int offset = __ offset();
 2397 
 2398   __ adr(lr, __ pc());
 2399   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2400 
 2401   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2402   __ end_a_stub();
 2403   return offset;
 2404 }
 2405 
 2406 // REQUIRED MATCHER CODE
 2407 
 2408 //=============================================================================
 2409 
 2410 const bool Matcher::match_rule_supported(int opcode) {
 2411   if (!has_match_rule(opcode))
 2412     return false;
 2413 
 2414   bool ret_value = true;
 2415   switch (opcode) {
 2416     case Op_OnSpinWait:
 2417       return VM_Version::supports_on_spin_wait();
 2418     case Op_CacheWB:
 2419     case Op_CacheWBPreSync:
 2420     case Op_CacheWBPostSync:
 2421       if (!VM_Version::supports_data_cache_line_flush()) {
 2422         ret_value = false;
 2423       }
 2424       break;
 2425     case Op_LoadVectorMasked:
 2426     case Op_StoreVectorMasked:
 2427     case Op_LoadVectorGatherMasked:
 2428     case Op_StoreVectorScatterMasked:
 2429     case Op_MaskAll:
 2430     case Op_AndVMask:
 2431     case Op_OrVMask:
 2432     case Op_XorVMask:
 2433       if (UseSVE == 0) {
 2434         ret_value = false;
 2435       }
 2436       break;
 2437     case Op_PopCountI:
 2438     case Op_PopCountL:
 2439     case Op_PopCountVI:
 2440     case Op_PopCountVL:
 2441       if (!UsePopCountInstruction) {
 2442         ret_value = false;
 2443       }
 2444       break;
 2445   }
 2446 
 2447   return ret_value; // Per default match rules are supported.
 2448 }
 2449 
 2450 // Identify extra cases that we might want to provide match rules for vector nodes and
 2451 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2452 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2453   if (!match_rule_supported(opcode)) {
 2454     return false;
 2455   }
 2456   int bit_size = vlen * type2aelembytes(bt) * 8;
 2457   if (UseSVE == 0 && bit_size > 128) {
 2458     return false;
 2459   }
 2460   if (UseSVE > 0) {
 2461     return op_sve_supported(opcode, vlen, bt);
 2462   } else { // NEON
 2463     // Special cases
 2464     switch (opcode) {
 2465     case Op_VectorMaskCmp:
 2466       if (vlen < 2 || bit_size < 64) {
 2467         return false;
 2468       }
 2469       break;
 2470     case Op_MulAddVS2VI:
 2471       if (bit_size < 128) {
 2472         return false;
 2473       }
 2474       break;
 2475     case Op_MulVL:
 2476       return false;
 2477     case Op_VectorLoadShuffle:
 2478     case Op_VectorRearrange:
 2479       if (vlen < 4) {
 2480         return false;
 2481       }
 2482       break;
 2483     case Op_LoadVectorGather:
 2484     case Op_StoreVectorScatter:
 2485     case Op_CompressV:
 2486     case Op_CompressM:
 2487     case Op_ExpandV:
 2488       return false;
 2489     default:
 2490       break;
 2491     }
 2492   }
 2493   return vector_size_supported(bt, vlen);
 2494 }
 2495 
 2496 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
 2497   // Only SVE supports masked operations.
 2498   if (UseSVE == 0) {
 2499     return false;
 2500   }
 2501   return match_rule_supported(opcode) &&
 2502          masked_op_sve_supported(opcode, vlen, bt);
 2503 }
 2504 
 2505 const RegMask* Matcher::predicate_reg_mask(void) {
 2506   return &_PR_REG_mask;
 2507 }
 2508 
 2509 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2510   return new TypeVectMask(elemTy, length);
 2511 }
 2512 
 2513 // Vector calling convention not yet implemented.
 2514 const bool Matcher::supports_vector_calling_convention(void) {
 2515   return false;
 2516 }
 2517 
 2518 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2519   Unimplemented();
 2520   return OptoRegPair(0, 0);
 2521 }
 2522 
 2523 // Is this branch offset short enough that a short branch can be used?
 2524 //
 2525 // NOTE: If the platform does not provide any short branch variants, then
 2526 //       this method should return false for offset 0.
 2527 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2528   // The passed offset is relative to address of the branch.
 2529 
 2530   return (-32768 <= offset && offset < 32768);
 2531 }
 2532 
 2533 // Vector width in bytes.
 2534 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2535   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2536   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2537   // Minimum 2 values in vector
 2538   if (size < 2*type2aelembytes(bt)) size = 0;
 2539   // But never < 4
 2540   if (size < 4) size = 0;
 2541   return size;
 2542 }
 2543 
 2544 // Limits on vector size (number of elements) loaded into vector.
 2545 const int Matcher::max_vector_size(const BasicType bt) {
 2546   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2547 }
 2548 
 2549 const int Matcher::min_vector_size(const BasicType bt) {
 2550   int max_size = max_vector_size(bt);
 2551   // Limit the min vector size to 8 bytes.
 2552   int size = 8 / type2aelembytes(bt);
 2553   if (bt == T_BYTE) {
 2554     // To support vector api shuffle/rearrange.
 2555     size = 4;
 2556   } else if (bt == T_BOOLEAN) {
 2557     // To support vector api load/store mask.
 2558     size = 2;
 2559   }
 2560   if (size < 2) size = 2;
 2561   return MIN2(size, max_size);
 2562 }
 2563 
 2564 // Actual max scalable vector register length.
 2565 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2566   return Matcher::max_vector_size(bt);
 2567 }
 2568 
 2569 // Vector ideal reg.
 2570 const uint Matcher::vector_ideal_reg(int len) {
 2571   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2572     return Op_VecA;
 2573   }
 2574   switch(len) {
 2575     // For 16-bit/32-bit mask vector, reuse VecD.
 2576     case  2:
 2577     case  4:
 2578     case  8: return Op_VecD;
 2579     case 16: return Op_VecX;
 2580   }
 2581   ShouldNotReachHere();
 2582   return 0;
 2583 }
 2584 
 2585 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2586   ShouldNotReachHere(); // generic vector operands not supported
 2587   return NULL;
 2588 }
 2589 
 2590 bool Matcher::is_reg2reg_move(MachNode* m) {
 2591   ShouldNotReachHere();  // generic vector operands not supported
 2592   return false;
 2593 }
 2594 
 2595 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2596   ShouldNotReachHere();  // generic vector operands not supported
 2597   return false;
 2598 }
 2599 
 2600 // Return whether or not this register is ever used as an argument.
 2601 // This function is used on startup to build the trampoline stubs in
 2602 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2603 // call in the trampoline, and arguments in those registers not be
 2604 // available to the callee.
 2605 bool Matcher::can_be_java_arg(int reg)
 2606 {
 2607   return
 2608     reg ==  R0_num || reg == R0_H_num ||
 2609     reg ==  R1_num || reg == R1_H_num ||
 2610     reg ==  R2_num || reg == R2_H_num ||
 2611     reg ==  R3_num || reg == R3_H_num ||
 2612     reg ==  R4_num || reg == R4_H_num ||
 2613     reg ==  R5_num || reg == R5_H_num ||
 2614     reg ==  R6_num || reg == R6_H_num ||
 2615     reg ==  R7_num || reg == R7_H_num ||
 2616     reg ==  V0_num || reg == V0_H_num ||
 2617     reg ==  V1_num || reg == V1_H_num ||
 2618     reg ==  V2_num || reg == V2_H_num ||
 2619     reg ==  V3_num || reg == V3_H_num ||
 2620     reg ==  V4_num || reg == V4_H_num ||
 2621     reg ==  V5_num || reg == V5_H_num ||
 2622     reg ==  V6_num || reg == V6_H_num ||
 2623     reg ==  V7_num || reg == V7_H_num;
 2624 }
 2625 
 2626 bool Matcher::is_spillable_arg(int reg)
 2627 {
 2628   return can_be_java_arg(reg);
 2629 }
 2630 
 2631 uint Matcher::int_pressure_limit()
 2632 {
 2633   // JDK-8183543: When taking the number of available registers as int
 2634   // register pressure threshold, the jtreg test:
 2635   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2636   // failed due to C2 compilation failure with
 2637   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2638   //
 2639   // A derived pointer is live at CallNode and then is flagged by RA
 2640   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2641   // derived pointers and lastly fail to spill after reaching maximum
 2642   // number of iterations. Lowering the default pressure threshold to
 2643   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2644   // a high register pressure area of the code so that split_DEF can
 2645   // generate DefinitionSpillCopy for the derived pointer.
 2646   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2647   if (!PreserveFramePointer) {
 2648     // When PreserveFramePointer is off, frame pointer is allocatable,
 2649     // but different from other SOC registers, it is excluded from
 2650     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2651     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2652     // See check_pressure_at_fatproj().
 2653     default_int_pressure_threshold--;
 2654   }
 2655   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2656 }
 2657 
 2658 uint Matcher::float_pressure_limit()
 2659 {
 2660   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2661   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2662 }
 2663 
 2664 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2665   return false;
 2666 }
 2667 
 2668 RegMask Matcher::divI_proj_mask() {
 2669   ShouldNotReachHere();
 2670   return RegMask();
 2671 }
 2672 
 2673 // Register for MODI projection of divmodI.
 2674 RegMask Matcher::modI_proj_mask() {
 2675   ShouldNotReachHere();
 2676   return RegMask();
 2677 }
 2678 
 2679 // Register for DIVL projection of divmodL.
 2680 RegMask Matcher::divL_proj_mask() {
 2681   ShouldNotReachHere();
 2682   return RegMask();
 2683 }
 2684 
 2685 // Register for MODL projection of divmodL.
 2686 RegMask Matcher::modL_proj_mask() {
 2687   ShouldNotReachHere();
 2688   return RegMask();
 2689 }
 2690 
 2691 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2692   return FP_REG_mask();
 2693 }
 2694 
 2695 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2696   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2697     Node* u = addp->fast_out(i);
 2698     if (u->is_LoadStore()) {
 2699       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2700       // instructions) only take register indirect as an operand, so
 2701       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2702       // must fail.
 2703       return false;
 2704     }
 2705     if (u->is_Mem()) {
 2706       int opsize = u->as_Mem()->memory_size();
 2707       assert(opsize > 0, "unexpected memory operand size");
 2708       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2709         return false;
 2710       }
 2711     }
 2712   }
 2713   return true;
 2714 }
 2715 
 2716 bool can_combine_with_imm(Node* binary_node, Node* replicate_node) {
 2717   if (UseSVE == 0 || !VectorNode::is_invariant_vector(replicate_node)){
 2718     return false;
 2719   }
 2720   Node* imm_node = replicate_node->in(1);
 2721   if (!imm_node->is_Con()) {
 2722     return false;
 2723   }
 2724 
 2725   const Type* t = imm_node->bottom_type();
 2726   if (!(t->isa_int() || t->isa_long())) {
 2727     return false;
 2728   }
 2729 
 2730   switch (binary_node->Opcode()) {
 2731   case Op_AndV:
 2732   case Op_OrV:
 2733   case Op_XorV: {
 2734     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(binary_node));
 2735     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2736     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2737   }
 2738   case Op_AddVB:
 2739     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2740   case Op_AddVS:
 2741   case Op_AddVI:
 2742     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2743   case Op_AddVL:
 2744     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2745   default:
 2746     return false;
 2747   }
 2748 }
 2749 
 2750 bool is_vector_arith_imm_pattern(Node* n, Node* m) {
 2751   if (n != NULL && m != NULL) {
 2752     return can_combine_with_imm(n, m);
 2753   }
 2754   return false;
 2755 }
 2756 
 2757 // Should the matcher clone input 'm' of node 'n'?
 2758 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2759   // ShiftV src (ShiftCntV con)
 2760   // Binary src (Replicate con)
 2761   if (is_vshift_con_pattern(n, m) || is_vector_arith_imm_pattern(n, m)) {
 2762     mstack.push(m, Visit);
 2763     return true;
 2764   }
 2765 
 2766   return false;
 2767 }
 2768 
 2769 // Should the Matcher clone shifts on addressing modes, expecting them
 2770 // to be subsumed into complex addressing expressions or compute them
 2771 // into registers?
 2772 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2773   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2774     return true;
 2775   }
 2776 
 2777   Node *off = m->in(AddPNode::Offset);
 2778   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2779       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2780       // Are there other uses besides address expressions?
 2781       !is_visited(off)) {
 2782     address_visited.set(off->_idx); // Flag as address_visited
 2783     mstack.push(off->in(2), Visit);
 2784     Node *conv = off->in(1);
 2785     if (conv->Opcode() == Op_ConvI2L &&
 2786         // Are there other uses besides address expressions?
 2787         !is_visited(conv)) {
 2788       address_visited.set(conv->_idx); // Flag as address_visited
 2789       mstack.push(conv->in(1), Pre_Visit);
 2790     } else {
 2791       mstack.push(conv, Pre_Visit);
 2792     }
 2793     address_visited.test_set(m->_idx); // Flag as address_visited
 2794     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2795     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2796     return true;
 2797   } else if (off->Opcode() == Op_ConvI2L &&
 2798              // Are there other uses besides address expressions?
 2799              !is_visited(off)) {
 2800     address_visited.test_set(m->_idx); // Flag as address_visited
 2801     address_visited.set(off->_idx); // Flag as address_visited
 2802     mstack.push(off->in(1), Pre_Visit);
 2803     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2804     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2805     return true;
 2806   }
 2807   return false;
 2808 }
 2809 
 2810 bool Parse::do_one_bytecode_targeted() {
 2811   return false;
 2812 }
 2813 
 2814 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2815   C2_MacroAssembler _masm(&cbuf);                                       \
 2816   {                                                                     \
 2817     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2818     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2819     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2820     __ INSN(REG, as_Register(BASE));                                    \
 2821   }
 2822 
 2823 
 2824 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2825   {
 2826     Address::extend scale;
 2827 
 2828     // Hooboy, this is fugly.  We need a way to communicate to the
 2829     // encoder that the index needs to be sign extended, so we have to
 2830     // enumerate all the cases.
 2831     switch (opcode) {
 2832     case INDINDEXSCALEDI2L:
 2833     case INDINDEXSCALEDI2LN:
 2834     case INDINDEXI2L:
 2835     case INDINDEXI2LN:
 2836       scale = Address::sxtw(size);
 2837       break;
 2838     default:
 2839       scale = Address::lsl(size);
 2840     }
 2841 
 2842     if (index == -1) {
 2843       return Address(base, disp);
 2844     } else {
 2845       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2846       return Address(base, as_Register(index), scale);
 2847     }
 2848   }
 2849 
 2850 
 2851 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2852 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2853 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2854 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2855                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2856 
 2857   // Used for all non-volatile memory accesses.  The use of
 2858   // $mem->opcode() to discover whether this pattern uses sign-extended
 2859   // offsets is something of a kludge.
 2860   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2861                         Register reg, int opcode,
 2862                         Register base, int index, int scale, int disp,
 2863                         int size_in_memory)
 2864   {
 2865     Address addr = mem2address(opcode, base, index, scale, disp);
 2866     if (addr.getMode() == Address::base_plus_offset) {
 2867       /* If we get an out-of-range offset it is a bug in the compiler,
 2868          so we assert here. */
 2869       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2870              "c2 compiler bug");
 2871       /* Fix up any out-of-range offsets. */
 2872       assert_different_registers(rscratch1, base);
 2873       assert_different_registers(rscratch1, reg);
 2874       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2875     }
 2876     (masm.*insn)(reg, addr);
 2877   }
 2878 
 2879   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2880                         FloatRegister reg, int opcode,
 2881                         Register base, int index, int size, int disp,
 2882                         int size_in_memory)
 2883   {
 2884     Address::extend scale;
 2885 
 2886     switch (opcode) {
 2887     case INDINDEXSCALEDI2L:
 2888     case INDINDEXSCALEDI2LN:
 2889       scale = Address::sxtw(size);
 2890       break;
 2891     default:
 2892       scale = Address::lsl(size);
 2893     }
 2894 
 2895     if (index == -1) {
 2896       /* If we get an out-of-range offset it is a bug in the compiler,
 2897          so we assert here. */
 2898       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2899       /* Fix up any out-of-range offsets. */
 2900       assert_different_registers(rscratch1, base);
 2901       Address addr = Address(base, disp);
 2902       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2903       (masm.*insn)(reg, addr);
 2904     } else {
 2905       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2906       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2907     }
 2908   }
 2909 
 2910   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2911                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2912                         int opcode, Register base, int index, int size, int disp)
 2913   {
 2914     if (index == -1) {
 2915       (masm.*insn)(reg, T, Address(base, disp));
 2916     } else {
 2917       assert(disp == 0, "unsupported address mode");
 2918       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2919     }
 2920   }
 2921 
 2922 %}
 2923 
 2924 
 2925 
 2926 //----------ENCODING BLOCK-----------------------------------------------------
 2927 // This block specifies the encoding classes used by the compiler to
 2928 // output byte streams.  Encoding classes are parameterized macros
 2929 // used by Machine Instruction Nodes in order to generate the bit
 2930 // encoding of the instruction.  Operands specify their base encoding
 2931 // interface with the interface keyword.  There are currently
 2932 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2933 // COND_INTER.  REG_INTER causes an operand to generate a function
 2934 // which returns its register number when queried.  CONST_INTER causes
 2935 // an operand to generate a function which returns the value of the
 2936 // constant when queried.  MEMORY_INTER causes an operand to generate
 2937 // four functions which return the Base Register, the Index Register,
 2938 // the Scale Value, and the Offset Value of the operand when queried.
 2939 // COND_INTER causes an operand to generate six functions which return
 2940 // the encoding code (ie - encoding bits for the instruction)
 2941 // associated with each basic boolean condition for a conditional
 2942 // instruction.
 2943 //
 2944 // Instructions specify two basic values for encoding.  Again, a
 2945 // function is available to check if the constant displacement is an
 2946 // oop. They use the ins_encode keyword to specify their encoding
 2947 // classes (which must be a sequence of enc_class names, and their
 2948 // parameters, specified in the encoding block), and they use the
 2949 // opcode keyword to specify, in order, their primary, secondary, and
 2950 // tertiary opcode.  Only the opcode sections which a particular
 2951 // instruction needs for encoding need to be specified.
 2952 encode %{
 2953   // Build emit functions for each basic byte or larger field in the
 2954   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2955   // from C++ code in the enc_class source block.  Emit functions will
 2956   // live in the main source block for now.  In future, we can
 2957   // generalize this by adding a syntax that specifies the sizes of
 2958   // fields in an order, so that the adlc can build the emit functions
 2959   // automagically
 2960 
 2961   // catch all for unimplemented encodings
 2962   enc_class enc_unimplemented %{
 2963     C2_MacroAssembler _masm(&cbuf);
 2964     __ unimplemented("C2 catch all");
 2965   %}
 2966 
 2967   // BEGIN Non-volatile memory access
 2968 
 2969   // This encoding class is generated automatically from ad_encode.m4.
 2970   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2971   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2972     Register dst_reg = as_Register($dst$$reg);
 2973     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2974                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2975   %}
 2976 
 2977   // This encoding class is generated automatically from ad_encode.m4.
 2978   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2979   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2980     Register dst_reg = as_Register($dst$$reg);
 2981     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2982                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2983   %}
 2984 
 2985   // This encoding class is generated automatically from ad_encode.m4.
 2986   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2987   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2988     Register dst_reg = as_Register($dst$$reg);
 2989     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2990                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2991   %}
 2992 
 2993   // This encoding class is generated automatically from ad_encode.m4.
 2994   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2995   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2996     Register dst_reg = as_Register($dst$$reg);
 2997     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2998                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2999   %}
 3000 
 3001   // This encoding class is generated automatically from ad_encode.m4.
 3002   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3003   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 3004     Register dst_reg = as_Register($dst$$reg);
 3005     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 3006                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3007   %}
 3008 
 3009   // This encoding class is generated automatically from ad_encode.m4.
 3010   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3011   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 3012     Register dst_reg = as_Register($dst$$reg);
 3013     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 3014                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3015   %}
 3016 
 3017   // This encoding class is generated automatically from ad_encode.m4.
 3018   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3019   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 3020     Register dst_reg = as_Register($dst$$reg);
 3021     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3022                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3023   %}
 3024 
 3025   // This encoding class is generated automatically from ad_encode.m4.
 3026   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3027   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 3028     Register dst_reg = as_Register($dst$$reg);
 3029     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3030                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3031   %}
 3032 
 3033   // This encoding class is generated automatically from ad_encode.m4.
 3034   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3035   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 3036     Register dst_reg = as_Register($dst$$reg);
 3037     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3038                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3039   %}
 3040 
 3041   // This encoding class is generated automatically from ad_encode.m4.
 3042   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3043   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 3044     Register dst_reg = as_Register($dst$$reg);
 3045     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3046                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3047   %}
 3048 
 3049   // This encoding class is generated automatically from ad_encode.m4.
 3050   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3051   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 3052     Register dst_reg = as_Register($dst$$reg);
 3053     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 3054                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3055   %}
 3056 
 3057   // This encoding class is generated automatically from ad_encode.m4.
 3058   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3059   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 3060     Register dst_reg = as_Register($dst$$reg);
 3061     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 3062                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3063   %}
 3064 
 3065   // This encoding class is generated automatically from ad_encode.m4.
 3066   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3067   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3068     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3069     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3070                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3071   %}
 3072 
 3073   // This encoding class is generated automatically from ad_encode.m4.
 3074   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3075   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3076     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3077     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3078                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3079   %}
 3080 
 3081   // This encoding class is generated automatically from ad_encode.m4.
 3082   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3083   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3084     Register src_reg = as_Register($src$$reg);
 3085     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 3086                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3087   %}
 3088 
 3089   // This encoding class is generated automatically from ad_encode.m4.
 3090   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3091   enc_class aarch64_enc_strb0(memory1 mem) %{
 3092     C2_MacroAssembler _masm(&cbuf);
 3093     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3094                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3095   %}
 3096 
 3097   // This encoding class is generated automatically from ad_encode.m4.
 3098   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3099   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3100     Register src_reg = as_Register($src$$reg);
 3101     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3102                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3103   %}
 3104 
 3105   // This encoding class is generated automatically from ad_encode.m4.
 3106   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3107   enc_class aarch64_enc_strh0(memory2 mem) %{
 3108     C2_MacroAssembler _masm(&cbuf);
 3109     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3110                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3111   %}
 3112 
 3113   // This encoding class is generated automatically from ad_encode.m4.
 3114   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3115   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3116     Register src_reg = as_Register($src$$reg);
 3117     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3118                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3119   %}
 3120 
 3121   // This encoding class is generated automatically from ad_encode.m4.
 3122   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3123   enc_class aarch64_enc_strw0(memory4 mem) %{
 3124     C2_MacroAssembler _masm(&cbuf);
 3125     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3126                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3127   %}
 3128 
 3129   // This encoding class is generated automatically from ad_encode.m4.
 3130   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3131   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3132     Register src_reg = as_Register($src$$reg);
 3133     // we sometimes get asked to store the stack pointer into the
 3134     // current thread -- we cannot do that directly on AArch64
 3135     if (src_reg == r31_sp) {
 3136       C2_MacroAssembler _masm(&cbuf);
 3137       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3138       __ mov(rscratch2, sp);
 3139       src_reg = rscratch2;
 3140     }
 3141     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3142                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3143   %}
 3144 
 3145   // This encoding class is generated automatically from ad_encode.m4.
 3146   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3147   enc_class aarch64_enc_str0(memory8 mem) %{
 3148     C2_MacroAssembler _masm(&cbuf);
 3149     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3150                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3151   %}
 3152 
 3153   // This encoding class is generated automatically from ad_encode.m4.
 3154   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3155   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3156     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3157     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3158                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3159   %}
 3160 
 3161   // This encoding class is generated automatically from ad_encode.m4.
 3162   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3163   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3164     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3165     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3166                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3167   %}
 3168 
 3169   // This encoding class is generated automatically from ad_encode.m4.
 3170   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3171   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3172       C2_MacroAssembler _masm(&cbuf);
 3173       __ membar(Assembler::StoreStore);
 3174       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3175                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3176   %}
 3177 
 3178   // END Non-volatile memory access
 3179 
 3180   // Vector loads and stores
 3181   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3182     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3183     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3184        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3185   %}
 3186 
 3187   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3188     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3189     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3190        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3191   %}
 3192 
 3193   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3194     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3195     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3196        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3197   %}
 3198 
 3199   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3200     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3201     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3202        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3203   %}
 3204 
 3205   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3206     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3207     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3208        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3209   %}
 3210 
 3211   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3212     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3213     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3214        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3215   %}
 3216 
 3217   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3218     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3219     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3220        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3221   %}
 3222 
 3223   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3224     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3225     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3226        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3227   %}
 3228 
 3229   // volatile loads and stores
 3230 
 3231   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3232     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3233                  rscratch1, stlrb);
 3234   %}
 3235 
 3236   enc_class aarch64_enc_stlrb0(memory mem) %{
 3237     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3238                  rscratch1, stlrb);
 3239   %}
 3240 
 3241   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3242     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3243                  rscratch1, stlrh);
 3244   %}
 3245 
 3246   enc_class aarch64_enc_stlrh0(memory mem) %{
 3247     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3248                  rscratch1, stlrh);
 3249   %}
 3250 
 3251   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3252     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3253                  rscratch1, stlrw);
 3254   %}
 3255 
 3256   enc_class aarch64_enc_stlrw0(memory mem) %{
 3257     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3258                  rscratch1, stlrw);
 3259   %}
 3260 
 3261   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3262     Register dst_reg = as_Register($dst$$reg);
 3263     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3264              rscratch1, ldarb);
 3265     __ sxtbw(dst_reg, dst_reg);
 3266   %}
 3267 
 3268   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3269     Register dst_reg = as_Register($dst$$reg);
 3270     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3271              rscratch1, ldarb);
 3272     __ sxtb(dst_reg, dst_reg);
 3273   %}
 3274 
 3275   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3276     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3277              rscratch1, ldarb);
 3278   %}
 3279 
 3280   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3281     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3282              rscratch1, ldarb);
 3283   %}
 3284 
 3285   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3286     Register dst_reg = as_Register($dst$$reg);
 3287     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3288              rscratch1, ldarh);
 3289     __ sxthw(dst_reg, dst_reg);
 3290   %}
 3291 
 3292   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3293     Register dst_reg = as_Register($dst$$reg);
 3294     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3295              rscratch1, ldarh);
 3296     __ sxth(dst_reg, dst_reg);
 3297   %}
 3298 
 3299   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3300     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3301              rscratch1, ldarh);
 3302   %}
 3303 
 3304   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3305     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3306              rscratch1, ldarh);
 3307   %}
 3308 
 3309   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3310     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3311              rscratch1, ldarw);
 3312   %}
 3313 
 3314   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3315     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3316              rscratch1, ldarw);
 3317   %}
 3318 
 3319   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3320     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3321              rscratch1, ldar);
 3322   %}
 3323 
 3324   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3325     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3326              rscratch1, ldarw);
 3327     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3328   %}
 3329 
 3330   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3331     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3332              rscratch1, ldar);
 3333     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3334   %}
 3335 
 3336   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3337     Register src_reg = as_Register($src$$reg);
 3338     // we sometimes get asked to store the stack pointer into the
 3339     // current thread -- we cannot do that directly on AArch64
 3340     if (src_reg == r31_sp) {
 3341       C2_MacroAssembler _masm(&cbuf);
 3342       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3343       __ mov(rscratch2, sp);
 3344       src_reg = rscratch2;
 3345     }
 3346     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3347                  rscratch1, stlr);
 3348   %}
 3349 
 3350   enc_class aarch64_enc_stlr0(memory mem) %{
 3351     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3352                  rscratch1, stlr);
 3353   %}
 3354 
 3355   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3356     {
 3357       C2_MacroAssembler _masm(&cbuf);
 3358       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3359       __ fmovs(rscratch2, src_reg);
 3360     }
 3361     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3362                  rscratch1, stlrw);
 3363   %}
 3364 
 3365   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3366     {
 3367       C2_MacroAssembler _masm(&cbuf);
 3368       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3369       __ fmovd(rscratch2, src_reg);
 3370     }
 3371     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3372                  rscratch1, stlr);
 3373   %}
 3374 
 3375   // synchronized read/update encodings
 3376 
 3377   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3378     C2_MacroAssembler _masm(&cbuf);
 3379     Register dst_reg = as_Register($dst$$reg);
 3380     Register base = as_Register($mem$$base);
 3381     int index = $mem$$index;
 3382     int scale = $mem$$scale;
 3383     int disp = $mem$$disp;
 3384     if (index == -1) {
 3385        if (disp != 0) {
 3386         __ lea(rscratch1, Address(base, disp));
 3387         __ ldaxr(dst_reg, rscratch1);
 3388       } else {
 3389         // TODO
 3390         // should we ever get anything other than this case?
 3391         __ ldaxr(dst_reg, base);
 3392       }
 3393     } else {
 3394       Register index_reg = as_Register(index);
 3395       if (disp == 0) {
 3396         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3397         __ ldaxr(dst_reg, rscratch1);
 3398       } else {
 3399         __ lea(rscratch1, Address(base, disp));
 3400         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3401         __ ldaxr(dst_reg, rscratch1);
 3402       }
 3403     }
 3404   %}
 3405 
 3406   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3407     C2_MacroAssembler _masm(&cbuf);
 3408     Register src_reg = as_Register($src$$reg);
 3409     Register base = as_Register($mem$$base);
 3410     int index = $mem$$index;
 3411     int scale = $mem$$scale;
 3412     int disp = $mem$$disp;
 3413     if (index == -1) {
 3414        if (disp != 0) {
 3415         __ lea(rscratch2, Address(base, disp));
 3416         __ stlxr(rscratch1, src_reg, rscratch2);
 3417       } else {
 3418         // TODO
 3419         // should we ever get anything other than this case?
 3420         __ stlxr(rscratch1, src_reg, base);
 3421       }
 3422     } else {
 3423       Register index_reg = as_Register(index);
 3424       if (disp == 0) {
 3425         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3426         __ stlxr(rscratch1, src_reg, rscratch2);
 3427       } else {
 3428         __ lea(rscratch2, Address(base, disp));
 3429         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3430         __ stlxr(rscratch1, src_reg, rscratch2);
 3431       }
 3432     }
 3433     __ cmpw(rscratch1, zr);
 3434   %}
 3435 
 3436   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3437     C2_MacroAssembler _masm(&cbuf);
 3438     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3439     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3440                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3441                /*weak*/ false, noreg);
 3442   %}
 3443 
 3444   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3445     C2_MacroAssembler _masm(&cbuf);
 3446     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3447     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3448                Assembler::word, /*acquire*/ false, /*release*/ true,
 3449                /*weak*/ false, noreg);
 3450   %}
 3451 
 3452   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3453     C2_MacroAssembler _masm(&cbuf);
 3454     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3455     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3456                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3457                /*weak*/ false, noreg);
 3458   %}
 3459 
 3460   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3461     C2_MacroAssembler _masm(&cbuf);
 3462     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3463     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3464                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3465                /*weak*/ false, noreg);
 3466   %}
 3467 
 3468 
 3469   // The only difference between aarch64_enc_cmpxchg and
 3470   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3471   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3472   // lock.
 3473   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3474     C2_MacroAssembler _masm(&cbuf);
 3475     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3476     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3477                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3478                /*weak*/ false, noreg);
 3479   %}
 3480 
 3481   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3482     C2_MacroAssembler _masm(&cbuf);
 3483     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3484     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3485                Assembler::word, /*acquire*/ true, /*release*/ true,
 3486                /*weak*/ false, noreg);
 3487   %}
 3488 
 3489   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3490     C2_MacroAssembler _masm(&cbuf);
 3491     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3492     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3493                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3494                /*weak*/ false, noreg);
 3495   %}
 3496 
 3497   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3498     C2_MacroAssembler _masm(&cbuf);
 3499     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3500     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3501                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3502                /*weak*/ false, noreg);
 3503   %}
 3504 
 3505   // auxiliary used for CompareAndSwapX to set result register
 3506   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3507     C2_MacroAssembler _masm(&cbuf);
 3508     Register res_reg = as_Register($res$$reg);
 3509     __ cset(res_reg, Assembler::EQ);
 3510   %}
 3511 
 3512   // prefetch encodings
 3513 
 3514   enc_class aarch64_enc_prefetchw(memory mem) %{
 3515     C2_MacroAssembler _masm(&cbuf);
 3516     Register base = as_Register($mem$$base);
 3517     int index = $mem$$index;
 3518     int scale = $mem$$scale;
 3519     int disp = $mem$$disp;
 3520     if (index == -1) {
 3521       __ prfm(Address(base, disp), PSTL1KEEP);
 3522     } else {
 3523       Register index_reg = as_Register(index);
 3524       if (disp == 0) {
 3525         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3526       } else {
 3527         __ lea(rscratch1, Address(base, disp));
 3528 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3529       }
 3530     }
 3531   %}
 3532 
 3533   /// mov envcodings
 3534 
 3535   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3536     C2_MacroAssembler _masm(&cbuf);
 3537     uint32_t con = (uint32_t)$src$$constant;
 3538     Register dst_reg = as_Register($dst$$reg);
 3539     if (con == 0) {
 3540       __ movw(dst_reg, zr);
 3541     } else {
 3542       __ movw(dst_reg, con);
 3543     }
 3544   %}
 3545 
 3546   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3547     C2_MacroAssembler _masm(&cbuf);
 3548     Register dst_reg = as_Register($dst$$reg);
 3549     uint64_t con = (uint64_t)$src$$constant;
 3550     if (con == 0) {
 3551       __ mov(dst_reg, zr);
 3552     } else {
 3553       __ mov(dst_reg, con);
 3554     }
 3555   %}
 3556 
 3557   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3558     C2_MacroAssembler _masm(&cbuf);
 3559     Register dst_reg = as_Register($dst$$reg);
 3560     address con = (address)$src$$constant;
 3561     if (con == NULL || con == (address)1) {
 3562       ShouldNotReachHere();
 3563     } else {
 3564       relocInfo::relocType rtype = $src->constant_reloc();
 3565       if (rtype == relocInfo::oop_type) {
 3566         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3567       } else if (rtype == relocInfo::metadata_type) {
 3568         __ mov_metadata(dst_reg, (Metadata*)con);
 3569       } else {
 3570         assert(rtype == relocInfo::none, "unexpected reloc type");
 3571         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3572           __ mov(dst_reg, con);
 3573         } else {
 3574           uint64_t offset;
 3575           __ adrp(dst_reg, con, offset);
 3576           __ add(dst_reg, dst_reg, offset);
 3577         }
 3578       }
 3579     }
 3580   %}
 3581 
 3582   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3583     C2_MacroAssembler _masm(&cbuf);
 3584     Register dst_reg = as_Register($dst$$reg);
 3585     __ mov(dst_reg, zr);
 3586   %}
 3587 
 3588   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3589     C2_MacroAssembler _masm(&cbuf);
 3590     Register dst_reg = as_Register($dst$$reg);
 3591     __ mov(dst_reg, (uint64_t)1);
 3592   %}
 3593 
 3594   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3595     C2_MacroAssembler _masm(&cbuf);
 3596     __ load_byte_map_base($dst$$Register);
 3597   %}
 3598 
 3599   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3600     C2_MacroAssembler _masm(&cbuf);
 3601     Register dst_reg = as_Register($dst$$reg);
 3602     address con = (address)$src$$constant;
 3603     if (con == NULL) {
 3604       ShouldNotReachHere();
 3605     } else {
 3606       relocInfo::relocType rtype = $src->constant_reloc();
 3607       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3608       __ set_narrow_oop(dst_reg, (jobject)con);
 3609     }
 3610   %}
 3611 
 3612   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3613     C2_MacroAssembler _masm(&cbuf);
 3614     Register dst_reg = as_Register($dst$$reg);
 3615     __ mov(dst_reg, zr);
 3616   %}
 3617 
 3618   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3619     C2_MacroAssembler _masm(&cbuf);
 3620     Register dst_reg = as_Register($dst$$reg);
 3621     address con = (address)$src$$constant;
 3622     if (con == NULL) {
 3623       ShouldNotReachHere();
 3624     } else {
 3625       relocInfo::relocType rtype = $src->constant_reloc();
 3626       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3627       __ set_narrow_klass(dst_reg, (Klass *)con);
 3628     }
 3629   %}
 3630 
 3631   // arithmetic encodings
 3632 
 3633   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3634     C2_MacroAssembler _masm(&cbuf);
 3635     Register dst_reg = as_Register($dst$$reg);
 3636     Register src_reg = as_Register($src1$$reg);
 3637     int32_t con = (int32_t)$src2$$constant;
 3638     // add has primary == 0, subtract has primary == 1
 3639     if ($primary) { con = -con; }
 3640     if (con < 0) {
 3641       __ subw(dst_reg, src_reg, -con);
 3642     } else {
 3643       __ addw(dst_reg, src_reg, con);
 3644     }
 3645   %}
 3646 
 3647   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3648     C2_MacroAssembler _masm(&cbuf);
 3649     Register dst_reg = as_Register($dst$$reg);
 3650     Register src_reg = as_Register($src1$$reg);
 3651     int32_t con = (int32_t)$src2$$constant;
 3652     // add has primary == 0, subtract has primary == 1
 3653     if ($primary) { con = -con; }
 3654     if (con < 0) {
 3655       __ sub(dst_reg, src_reg, -con);
 3656     } else {
 3657       __ add(dst_reg, src_reg, con);
 3658     }
 3659   %}
 3660 
 3661   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3662     C2_MacroAssembler _masm(&cbuf);
 3663    Register dst_reg = as_Register($dst$$reg);
 3664    Register src1_reg = as_Register($src1$$reg);
 3665    Register src2_reg = as_Register($src2$$reg);
 3666     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3667   %}
 3668 
 3669   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3670     C2_MacroAssembler _masm(&cbuf);
 3671    Register dst_reg = as_Register($dst$$reg);
 3672    Register src1_reg = as_Register($src1$$reg);
 3673    Register src2_reg = as_Register($src2$$reg);
 3674     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3675   %}
 3676 
 3677   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3678     C2_MacroAssembler _masm(&cbuf);
 3679    Register dst_reg = as_Register($dst$$reg);
 3680    Register src1_reg = as_Register($src1$$reg);
 3681    Register src2_reg = as_Register($src2$$reg);
 3682     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3683   %}
 3684 
 3685   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3686     C2_MacroAssembler _masm(&cbuf);
 3687    Register dst_reg = as_Register($dst$$reg);
 3688    Register src1_reg = as_Register($src1$$reg);
 3689    Register src2_reg = as_Register($src2$$reg);
 3690     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3691   %}
 3692 
 3693   // compare instruction encodings
 3694 
 3695   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3696     C2_MacroAssembler _masm(&cbuf);
 3697     Register reg1 = as_Register($src1$$reg);
 3698     Register reg2 = as_Register($src2$$reg);
 3699     __ cmpw(reg1, reg2);
 3700   %}
 3701 
 3702   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3703     C2_MacroAssembler _masm(&cbuf);
 3704     Register reg = as_Register($src1$$reg);
 3705     int32_t val = $src2$$constant;
 3706     if (val >= 0) {
 3707       __ subsw(zr, reg, val);
 3708     } else {
 3709       __ addsw(zr, reg, -val);
 3710     }
 3711   %}
 3712 
 3713   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3714     C2_MacroAssembler _masm(&cbuf);
 3715     Register reg1 = as_Register($src1$$reg);
 3716     uint32_t val = (uint32_t)$src2$$constant;
 3717     __ movw(rscratch1, val);
 3718     __ cmpw(reg1, rscratch1);
 3719   %}
 3720 
 3721   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3722     C2_MacroAssembler _masm(&cbuf);
 3723     Register reg1 = as_Register($src1$$reg);
 3724     Register reg2 = as_Register($src2$$reg);
 3725     __ cmp(reg1, reg2);
 3726   %}
 3727 
 3728   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3729     C2_MacroAssembler _masm(&cbuf);
 3730     Register reg = as_Register($src1$$reg);
 3731     int64_t val = $src2$$constant;
 3732     if (val >= 0) {
 3733       __ subs(zr, reg, val);
 3734     } else if (val != -val) {
 3735       __ adds(zr, reg, -val);
 3736     } else {
 3737     // aargh, Long.MIN_VALUE is a special case
 3738       __ orr(rscratch1, zr, (uint64_t)val);
 3739       __ subs(zr, reg, rscratch1);
 3740     }
 3741   %}
 3742 
 3743   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3744     C2_MacroAssembler _masm(&cbuf);
 3745     Register reg1 = as_Register($src1$$reg);
 3746     uint64_t val = (uint64_t)$src2$$constant;
 3747     __ mov(rscratch1, val);
 3748     __ cmp(reg1, rscratch1);
 3749   %}
 3750 
 3751   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3752     C2_MacroAssembler _masm(&cbuf);
 3753     Register reg1 = as_Register($src1$$reg);
 3754     Register reg2 = as_Register($src2$$reg);
 3755     __ cmp(reg1, reg2);
 3756   %}
 3757 
 3758   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3759     C2_MacroAssembler _masm(&cbuf);
 3760     Register reg1 = as_Register($src1$$reg);
 3761     Register reg2 = as_Register($src2$$reg);
 3762     __ cmpw(reg1, reg2);
 3763   %}
 3764 
 3765   enc_class aarch64_enc_testp(iRegP src) %{
 3766     C2_MacroAssembler _masm(&cbuf);
 3767     Register reg = as_Register($src$$reg);
 3768     __ cmp(reg, zr);
 3769   %}
 3770 
 3771   enc_class aarch64_enc_testn(iRegN src) %{
 3772     C2_MacroAssembler _masm(&cbuf);
 3773     Register reg = as_Register($src$$reg);
 3774     __ cmpw(reg, zr);
 3775   %}
 3776 
 3777   enc_class aarch64_enc_b(label lbl) %{
 3778     C2_MacroAssembler _masm(&cbuf);
 3779     Label *L = $lbl$$label;
 3780     __ b(*L);
 3781   %}
 3782 
 3783   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3784     C2_MacroAssembler _masm(&cbuf);
 3785     Label *L = $lbl$$label;
 3786     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3787   %}
 3788 
 3789   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3790     C2_MacroAssembler _masm(&cbuf);
 3791     Label *L = $lbl$$label;
 3792     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3793   %}
 3794 
 3795   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3796   %{
 3797      Register sub_reg = as_Register($sub$$reg);
 3798      Register super_reg = as_Register($super$$reg);
 3799      Register temp_reg = as_Register($temp$$reg);
 3800      Register result_reg = as_Register($result$$reg);
 3801 
 3802      Label miss;
 3803      C2_MacroAssembler _masm(&cbuf);
 3804      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3805                                      NULL, &miss,
 3806                                      /*set_cond_codes:*/ true);
 3807      if ($primary) {
 3808        __ mov(result_reg, zr);
 3809      }
 3810      __ bind(miss);
 3811   %}
 3812 
 3813   enc_class aarch64_enc_java_static_call(method meth) %{
 3814     C2_MacroAssembler _masm(&cbuf);
 3815 
 3816     address addr = (address)$meth$$method;
 3817     address call;
 3818     if (!_method) {
 3819       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3820       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3821       if (call == NULL) {
 3822         ciEnv::current()->record_failure("CodeCache is full");
 3823         return;
 3824       }
 3825     } else {
 3826       int method_index = resolved_method_index(cbuf);
 3827       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3828                                                   : static_call_Relocation::spec(method_index);
 3829       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3830       if (call == NULL) {
 3831         ciEnv::current()->record_failure("CodeCache is full");
 3832         return;
 3833       }
 3834       // Emit stub for static call
 3835       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3836       if (stub == NULL) {
 3837         ciEnv::current()->record_failure("CodeCache is full");
 3838         return;
 3839       }
 3840     }
 3841 
 3842     // Only non uncommon_trap calls need to reinitialize ptrue.
 3843     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3844       __ reinitialize_ptrue();
 3845     }
 3846   %}
 3847 
 3848   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3849     C2_MacroAssembler _masm(&cbuf);
 3850     int method_index = resolved_method_index(cbuf);
 3851     address call = __ ic_call((address)$meth$$method, method_index);
 3852     if (call == NULL) {
 3853       ciEnv::current()->record_failure("CodeCache is full");
 3854       return;
 3855     } else if (Compile::current()->max_vector_size() > 0) {
 3856       __ reinitialize_ptrue();
 3857     }
 3858   %}
 3859 
 3860   enc_class aarch64_enc_call_epilog() %{
 3861     C2_MacroAssembler _masm(&cbuf);
 3862     if (VerifyStackAtCalls) {
 3863       // Check that stack depth is unchanged: find majik cookie on stack
 3864       __ call_Unimplemented();
 3865     }
 3866   %}
 3867 
 3868   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3869     C2_MacroAssembler _masm(&cbuf);
 3870 
 3871     // some calls to generated routines (arraycopy code) are scheduled
 3872     // by C2 as runtime calls. if so we can call them using a br (they
 3873     // will be in a reachable segment) otherwise we have to use a blr
 3874     // which loads the absolute address into a register.
 3875     address entry = (address)$meth$$method;
 3876     CodeBlob *cb = CodeCache::find_blob(entry);
 3877     if (cb) {
 3878       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3879       if (call == NULL) {
 3880         ciEnv::current()->record_failure("CodeCache is full");
 3881         return;
 3882       }
 3883     } else {
 3884       Label retaddr;
 3885       __ adr(rscratch2, retaddr);
 3886       __ lea(rscratch1, RuntimeAddress(entry));
 3887       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3888       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3889       __ blr(rscratch1);
 3890       __ bind(retaddr);
 3891       __ add(sp, sp, 2 * wordSize);
 3892     }
 3893     if (Compile::current()->max_vector_size() > 0) {
 3894       __ reinitialize_ptrue();
 3895     }
 3896   %}
 3897 
 3898   enc_class aarch64_enc_rethrow() %{
 3899     C2_MacroAssembler _masm(&cbuf);
 3900     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3901   %}
 3902 
 3903   enc_class aarch64_enc_ret() %{
 3904     C2_MacroAssembler _masm(&cbuf);
 3905 #ifdef ASSERT
 3906     if (Compile::current()->max_vector_size() > 0) {
 3907       __ verify_ptrue();
 3908     }
 3909 #endif
 3910     __ ret(lr);
 3911   %}
 3912 
 3913   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3914     C2_MacroAssembler _masm(&cbuf);
 3915     Register target_reg = as_Register($jump_target$$reg);
 3916     __ br(target_reg);
 3917   %}
 3918 
 3919   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3920     C2_MacroAssembler _masm(&cbuf);
 3921     Register target_reg = as_Register($jump_target$$reg);
 3922     // exception oop should be in r0
 3923     // ret addr has been popped into lr
 3924     // callee expects it in r3
 3925     __ mov(r3, lr);
 3926     __ br(target_reg);
 3927   %}
 3928 
 3929   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3930     C2_MacroAssembler _masm(&cbuf);
 3931     Register oop = as_Register($object$$reg);
 3932     Register box = as_Register($box$$reg);
 3933     Register disp_hdr = as_Register($tmp$$reg);
 3934     Register tmp = as_Register($tmp2$$reg);
 3935     Label cont;
 3936     Label object_has_monitor;
 3937     Label cas_failed;
 3938 
 3939     assert_different_registers(oop, box, tmp, disp_hdr);
 3940 
 3941     // Load markWord from object into displaced_header.
 3942     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3943 
 3944     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3945       __ load_klass(tmp, oop);
 3946       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3947       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3948       __ br(Assembler::NE, cont);
 3949     }
 3950 
 3951     // Check for existing monitor
 3952     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3953 
 3954     if (!UseHeavyMonitors) {
 3955       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3956       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3957 
 3958       // Initialize the box. (Must happen before we update the object mark!)
 3959       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3960 
 3961       // Compare object markWord with an unlocked value (tmp) and if
 3962       // equal exchange the stack address of our box with object markWord.
 3963       // On failure disp_hdr contains the possibly locked markWord.
 3964       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3965                  /*release*/ true, /*weak*/ false, disp_hdr);
 3966       __ br(Assembler::EQ, cont);
 3967 
 3968       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3969 
 3970       // If the compare-and-exchange succeeded, then we found an unlocked
 3971       // object, will have now locked it will continue at label cont
 3972 
 3973       __ bind(cas_failed);
 3974       // We did not see an unlocked object so try the fast recursive case.
 3975 
 3976       // Check if the owner is self by comparing the value in the
 3977       // markWord of object (disp_hdr) with the stack pointer.
 3978       __ mov(rscratch1, sp);
 3979       __ sub(disp_hdr, disp_hdr, rscratch1);
 3980       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3981       // If condition is true we are cont and hence we can store 0 as the
 3982       // displaced header in the box, which indicates that it is a recursive lock.
 3983       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3984       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3985     } else {
 3986       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3987     }
 3988     __ b(cont);
 3989 
 3990     // Handle existing monitor.
 3991     __ bind(object_has_monitor);
 3992 
 3993     // The object's monitor m is unlocked iff m->owner == NULL,
 3994     // otherwise m->owner may contain a thread or a stack address.
 3995     //
 3996     // Try to CAS m->owner from NULL to current thread.
 3997     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3998     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3999                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 4000 
 4001     // Store a non-null value into the box to avoid looking like a re-entrant
 4002     // lock. The fast-path monitor unlock code checks for
 4003     // markWord::monitor_value so use markWord::unused_mark which has the
 4004     // relevant bit set, and also matches ObjectSynchronizer::enter.
 4005     __ mov(tmp, (address)markWord::unused_mark().value());
 4006     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 4007 
 4008     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 4009 
 4010     __ cmp(rscratch1, rthread);
 4011     __ br(Assembler::NE, cont); // Check for recursive locking
 4012 
 4013     // Recursive lock case
 4014     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 4015     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 4016 
 4017     __ bind(cont);
 4018     // flag == EQ indicates success
 4019     // flag == NE indicates failure
 4020   %}
 4021 
 4022   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 4023     C2_MacroAssembler _masm(&cbuf);
 4024     Register oop = as_Register($object$$reg);
 4025     Register box = as_Register($box$$reg);
 4026     Register disp_hdr = as_Register($tmp$$reg);
 4027     Register tmp = as_Register($tmp2$$reg);
 4028     Label cont;
 4029     Label object_has_monitor;
 4030 
 4031     assert_different_registers(oop, box, tmp, disp_hdr);
 4032 
 4033     if (!UseHeavyMonitors) {
 4034       // Find the lock address and load the displaced header from the stack.
 4035       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 4036 
 4037       // If the displaced header is 0, we have a recursive unlock.
 4038       __ cmp(disp_hdr, zr);
 4039       __ br(Assembler::EQ, cont);
 4040     }
 4041 
 4042     // Handle existing monitor.
 4043     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 4044     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 4045 
 4046     if (!UseHeavyMonitors) {
 4047       // Check if it is still a light weight lock, this is is true if we
 4048       // see the stack address of the basicLock in the markWord of the
 4049       // object.
 4050 
 4051       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 4052                  /*release*/ true, /*weak*/ false, tmp);
 4053     } else {
 4054       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 4055     }
 4056     __ b(cont);
 4057 
 4058     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 4059 
 4060     // Handle existing monitor.
 4061     __ bind(object_has_monitor);
 4062     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 4063     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 4064     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4065 
 4066     Label notRecursive;
 4067     __ cbz(disp_hdr, notRecursive);
 4068 
 4069     // Recursive lock
 4070     __ sub(disp_hdr, disp_hdr, 1u);
 4071     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4072     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 4073     __ b(cont);
 4074 
 4075     __ bind(notRecursive);
 4076     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 4077     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 4078     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 4079     __ cmp(rscratch1, zr); // Sets flags for result
 4080     __ cbnz(rscratch1, cont);
 4081     // need a release store here
 4082     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 4083     __ stlr(zr, tmp); // set unowned
 4084 
 4085     __ bind(cont);
 4086     // flag == EQ indicates success
 4087     // flag == NE indicates failure
 4088   %}
 4089 
 4090 %}
 4091 
 4092 //----------FRAME--------------------------------------------------------------
 4093 // Definition of frame structure and management information.
 4094 //
 4095 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4096 //                             |   (to get allocators register number
 4097 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4098 //  r   CALLER     |        |
 4099 //  o     |        +--------+      pad to even-align allocators stack-slot
 4100 //  w     V        |  pad0  |        numbers; owned by CALLER
 4101 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4102 //  h     ^        |   in   |  5
 4103 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4104 //  |     |        |        |  3
 4105 //  |     |        +--------+
 4106 //  V     |        | old out|      Empty on Intel, window on Sparc
 4107 //        |    old |preserve|      Must be even aligned.
 4108 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4109 //        |        |   in   |  3   area for Intel ret address
 4110 //     Owned by    |preserve|      Empty on Sparc.
 4111 //       SELF      +--------+
 4112 //        |        |  pad2  |  2   pad to align old SP
 4113 //        |        +--------+  1
 4114 //        |        | locks  |  0
 4115 //        |        +--------+----> OptoReg::stack0(), even aligned
 4116 //        |        |  pad1  | 11   pad to align new SP
 4117 //        |        +--------+
 4118 //        |        |        | 10
 4119 //        |        | spills |  9   spills
 4120 //        V        |        |  8   (pad0 slot for callee)
 4121 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4122 //        ^        |  out   |  7
 4123 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4124 //     Owned by    +--------+
 4125 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4126 //        |    new |preserve|      Must be even-aligned.
 4127 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4128 //        |        |        |
 4129 //
 4130 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4131 //         known from SELF's arguments and the Java calling convention.
 4132 //         Region 6-7 is determined per call site.
 4133 // Note 2: If the calling convention leaves holes in the incoming argument
 4134 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4135 //         are owned by the CALLEE.  Holes should not be necessary in the
 4136 //         incoming area, as the Java calling convention is completely under
 4137 //         the control of the AD file.  Doubles can be sorted and packed to
 4138 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 4139 //         varargs C calling conventions.
 4140 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4141 //         even aligned with pad0 as needed.
 4142 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4143 //           (the latter is true on Intel but is it false on AArch64?)
 4144 //         region 6-11 is even aligned; it may be padded out more so that
 4145 //         the region from SP to FP meets the minimum stack alignment.
 4146 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4147 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4148 //         SP meets the minimum alignment.
 4149 
 4150 frame %{
 4151   // These three registers define part of the calling convention
 4152   // between compiled code and the interpreter.
 4153 
 4154   // Inline Cache Register or Method for I2C.
 4155   inline_cache_reg(R12);
 4156 
 4157   // Number of stack slots consumed by locking an object
 4158   sync_stack_slots(2);
 4159 
 4160   // Compiled code's Frame Pointer
 4161   frame_pointer(R31);
 4162 
 4163   // Interpreter stores its frame pointer in a register which is
 4164   // stored to the stack by I2CAdaptors.
 4165   // I2CAdaptors convert from interpreted java to compiled java.
 4166   interpreter_frame_pointer(R29);
 4167 
 4168   // Stack alignment requirement
 4169   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4170 
 4171   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4172   // for calls to C.  Supports the var-args backing area for register parms.
 4173   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4174 
 4175   // The after-PROLOG location of the return address.  Location of
 4176   // return address specifies a type (REG or STACK) and a number
 4177   // representing the register number (i.e. - use a register name) or
 4178   // stack slot.
 4179   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4180   // Otherwise, it is above the locks and verification slot and alignment word
 4181   // TODO this may well be correct but need to check why that - 2 is there
 4182   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4183   // which folds in the space used for monitors
 4184   return_addr(STACK - 2 +
 4185               align_up((Compile::current()->in_preserve_stack_slots() +
 4186                         Compile::current()->fixed_slots()),
 4187                        stack_alignment_in_slots()));
 4188 
 4189   // Location of compiled Java return values.  Same as C for now.
 4190   return_value
 4191   %{
 4192     // TODO do we allow ideal_reg == Op_RegN???
 4193     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4194            "only return normal values");
 4195 
 4196     static const int lo[Op_RegL + 1] = { // enum name
 4197       0,                                 // Op_Node
 4198       0,                                 // Op_Set
 4199       R0_num,                            // Op_RegN
 4200       R0_num,                            // Op_RegI
 4201       R0_num,                            // Op_RegP
 4202       V0_num,                            // Op_RegF
 4203       V0_num,                            // Op_RegD
 4204       R0_num                             // Op_RegL
 4205     };
 4206 
 4207     static const int hi[Op_RegL + 1] = { // enum name
 4208       0,                                 // Op_Node
 4209       0,                                 // Op_Set
 4210       OptoReg::Bad,                      // Op_RegN
 4211       OptoReg::Bad,                      // Op_RegI
 4212       R0_H_num,                          // Op_RegP
 4213       OptoReg::Bad,                      // Op_RegF
 4214       V0_H_num,                          // Op_RegD
 4215       R0_H_num                           // Op_RegL
 4216     };
 4217 
 4218     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4219   %}
 4220 %}
 4221 
 4222 //----------ATTRIBUTES---------------------------------------------------------
 4223 //----------Operand Attributes-------------------------------------------------
 4224 op_attrib op_cost(1);        // Required cost attribute
 4225 
 4226 //----------Instruction Attributes---------------------------------------------
 4227 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4228 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4229 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4230                                 // a non-matching short branch variant
 4231                                 // of some long branch?
 4232 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4233                                 // be a power of 2) specifies the
 4234                                 // alignment that some part of the
 4235                                 // instruction (not necessarily the
 4236                                 // start) requires.  If > 1, a
 4237                                 // compute_padding() function must be
 4238                                 // provided for the instruction
 4239 
 4240 //----------OPERANDS-----------------------------------------------------------
 4241 // Operand definitions must precede instruction definitions for correct parsing
 4242 // in the ADLC because operands constitute user defined types which are used in
 4243 // instruction definitions.
 4244 
 4245 //----------Simple Operands----------------------------------------------------
 4246 
 4247 // Integer operands 32 bit
 4248 // 32 bit immediate
 4249 operand immI()
 4250 %{
 4251   match(ConI);
 4252 
 4253   op_cost(0);
 4254   format %{ %}
 4255   interface(CONST_INTER);
 4256 %}
 4257 
 4258 // 32 bit zero
 4259 operand immI0()
 4260 %{
 4261   predicate(n->get_int() == 0);
 4262   match(ConI);
 4263 
 4264   op_cost(0);
 4265   format %{ %}
 4266   interface(CONST_INTER);
 4267 %}
 4268 
 4269 // 32 bit unit increment
 4270 operand immI_1()
 4271 %{
 4272   predicate(n->get_int() == 1);
 4273   match(ConI);
 4274 
 4275   op_cost(0);
 4276   format %{ %}
 4277   interface(CONST_INTER);
 4278 %}
 4279 
 4280 // 32 bit unit decrement
 4281 operand immI_M1()
 4282 %{
 4283   predicate(n->get_int() == -1);
 4284   match(ConI);
 4285 
 4286   op_cost(0);
 4287   format %{ %}
 4288   interface(CONST_INTER);
 4289 %}
 4290 
 4291 // Shift values for add/sub extension shift
 4292 operand immIExt()
 4293 %{
 4294   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4295   match(ConI);
 4296 
 4297   op_cost(0);
 4298   format %{ %}
 4299   interface(CONST_INTER);
 4300 %}
 4301 
 4302 operand immI_gt_1()
 4303 %{
 4304   predicate(n->get_int() > 1);
 4305   match(ConI);
 4306 
 4307   op_cost(0);
 4308   format %{ %}
 4309   interface(CONST_INTER);
 4310 %}
 4311 
 4312 operand immI_le_4()
 4313 %{
 4314   predicate(n->get_int() <= 4);
 4315   match(ConI);
 4316 
 4317   op_cost(0);
 4318   format %{ %}
 4319   interface(CONST_INTER);
 4320 %}
 4321 
 4322 operand immI_31()
 4323 %{
 4324   predicate(n->get_int() == 31);
 4325   match(ConI);
 4326 
 4327   op_cost(0);
 4328   format %{ %}
 4329   interface(CONST_INTER);
 4330 %}
 4331 
 4332 operand immI_2()
 4333 %{
 4334   predicate(n->get_int() == 2);
 4335   match(ConI);
 4336 
 4337   op_cost(0);
 4338   format %{ %}
 4339   interface(CONST_INTER);
 4340 %}
 4341 
 4342 operand immI_4()
 4343 %{
 4344   predicate(n->get_int() == 4);
 4345   match(ConI);
 4346 
 4347   op_cost(0);
 4348   format %{ %}
 4349   interface(CONST_INTER);
 4350 %}
 4351 
 4352 operand immI_8()
 4353 %{
 4354   predicate(n->get_int() == 8);
 4355   match(ConI);
 4356 
 4357   op_cost(0);
 4358   format %{ %}
 4359   interface(CONST_INTER);
 4360 %}
 4361 
 4362 operand immI_16()
 4363 %{
 4364   predicate(n->get_int() == 16);
 4365   match(ConI);
 4366 
 4367   op_cost(0);
 4368   format %{ %}
 4369   interface(CONST_INTER);
 4370 %}
 4371 
 4372 operand immI_24()
 4373 %{
 4374   predicate(n->get_int() == 24);
 4375   match(ConI);
 4376 
 4377   op_cost(0);
 4378   format %{ %}
 4379   interface(CONST_INTER);
 4380 %}
 4381 
 4382 operand immI_32()
 4383 %{
 4384   predicate(n->get_int() == 32);
 4385   match(ConI);
 4386 
 4387   op_cost(0);
 4388   format %{ %}
 4389   interface(CONST_INTER);
 4390 %}
 4391 
 4392 operand immI_48()
 4393 %{
 4394   predicate(n->get_int() == 48);
 4395   match(ConI);
 4396 
 4397   op_cost(0);
 4398   format %{ %}
 4399   interface(CONST_INTER);
 4400 %}
 4401 
 4402 operand immI_56()
 4403 %{
 4404   predicate(n->get_int() == 56);
 4405   match(ConI);
 4406 
 4407   op_cost(0);
 4408   format %{ %}
 4409   interface(CONST_INTER);
 4410 %}
 4411 
 4412 operand immI_63()
 4413 %{
 4414   predicate(n->get_int() == 63);
 4415   match(ConI);
 4416 
 4417   op_cost(0);
 4418   format %{ %}
 4419   interface(CONST_INTER);
 4420 %}
 4421 
 4422 operand immI_64()
 4423 %{
 4424   predicate(n->get_int() == 64);
 4425   match(ConI);
 4426 
 4427   op_cost(0);
 4428   format %{ %}
 4429   interface(CONST_INTER);
 4430 %}
 4431 
 4432 operand immI_255()
 4433 %{
 4434   predicate(n->get_int() == 255);
 4435   match(ConI);
 4436 
 4437   op_cost(0);
 4438   format %{ %}
 4439   interface(CONST_INTER);
 4440 %}
 4441 
 4442 operand immI_65535()
 4443 %{
 4444   predicate(n->get_int() == 65535);
 4445   match(ConI);
 4446 
 4447   op_cost(0);
 4448   format %{ %}
 4449   interface(CONST_INTER);
 4450 %}
 4451 
 4452 operand immL_255()
 4453 %{
 4454   predicate(n->get_long() == 255L);
 4455   match(ConL);
 4456 
 4457   op_cost(0);
 4458   format %{ %}
 4459   interface(CONST_INTER);
 4460 %}
 4461 
 4462 operand immL_65535()
 4463 %{
 4464   predicate(n->get_long() == 65535L);
 4465   match(ConL);
 4466 
 4467   op_cost(0);
 4468   format %{ %}
 4469   interface(CONST_INTER);
 4470 %}
 4471 
 4472 operand immL_4294967295()
 4473 %{
 4474   predicate(n->get_long() == 4294967295L);
 4475   match(ConL);
 4476 
 4477   op_cost(0);
 4478   format %{ %}
 4479   interface(CONST_INTER);
 4480 %}
 4481 
 4482 operand immL_bitmask()
 4483 %{
 4484   predicate((n->get_long() != 0)
 4485             && ((n->get_long() & 0xc000000000000000l) == 0)
 4486             && is_power_of_2(n->get_long() + 1));
 4487   match(ConL);
 4488 
 4489   op_cost(0);
 4490   format %{ %}
 4491   interface(CONST_INTER);
 4492 %}
 4493 
 4494 operand immI_bitmask()
 4495 %{
 4496   predicate((n->get_int() != 0)
 4497             && ((n->get_int() & 0xc0000000) == 0)
 4498             && is_power_of_2(n->get_int() + 1));
 4499   match(ConI);
 4500 
 4501   op_cost(0);
 4502   format %{ %}
 4503   interface(CONST_INTER);
 4504 %}
 4505 
 4506 operand immL_positive_bitmaskI()
 4507 %{
 4508   predicate((n->get_long() != 0)
 4509             && ((julong)n->get_long() < 0x80000000ULL)
 4510             && is_power_of_2(n->get_long() + 1));
 4511   match(ConL);
 4512 
 4513   op_cost(0);
 4514   format %{ %}
 4515   interface(CONST_INTER);
 4516 %}
 4517 
 4518 // Scale values for scaled offset addressing modes (up to long but not quad)
 4519 operand immIScale()
 4520 %{
 4521   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4522   match(ConI);
 4523 
 4524   op_cost(0);
 4525   format %{ %}
 4526   interface(CONST_INTER);
 4527 %}
 4528 
 4529 // 26 bit signed offset -- for pc-relative branches
 4530 operand immI26()
 4531 %{
 4532   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4533   match(ConI);
 4534 
 4535   op_cost(0);
 4536   format %{ %}
 4537   interface(CONST_INTER);
 4538 %}
 4539 
 4540 // 19 bit signed offset -- for pc-relative loads
 4541 operand immI19()
 4542 %{
 4543   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4544   match(ConI);
 4545 
 4546   op_cost(0);
 4547   format %{ %}
 4548   interface(CONST_INTER);
 4549 %}
 4550 
 4551 // 12 bit unsigned offset -- for base plus immediate loads
 4552 operand immIU12()
 4553 %{
 4554   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4555   match(ConI);
 4556 
 4557   op_cost(0);
 4558   format %{ %}
 4559   interface(CONST_INTER);
 4560 %}
 4561 
 4562 operand immLU12()
 4563 %{
 4564   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4565   match(ConL);
 4566 
 4567   op_cost(0);
 4568   format %{ %}
 4569   interface(CONST_INTER);
 4570 %}
 4571 
 4572 // Offset for scaled or unscaled immediate loads and stores
 4573 operand immIOffset()
 4574 %{
 4575   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4576   match(ConI);
 4577 
 4578   op_cost(0);
 4579   format %{ %}
 4580   interface(CONST_INTER);
 4581 %}
 4582 
 4583 operand immIOffset1()
 4584 %{
 4585   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4586   match(ConI);
 4587 
 4588   op_cost(0);
 4589   format %{ %}
 4590   interface(CONST_INTER);
 4591 %}
 4592 
 4593 operand immIOffset2()
 4594 %{
 4595   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4596   match(ConI);
 4597 
 4598   op_cost(0);
 4599   format %{ %}
 4600   interface(CONST_INTER);
 4601 %}
 4602 
 4603 operand immIOffset4()
 4604 %{
 4605   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4606   match(ConI);
 4607 
 4608   op_cost(0);
 4609   format %{ %}
 4610   interface(CONST_INTER);
 4611 %}
 4612 
 4613 operand immIOffset8()
 4614 %{
 4615   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4616   match(ConI);
 4617 
 4618   op_cost(0);
 4619   format %{ %}
 4620   interface(CONST_INTER);
 4621 %}
 4622 
 4623 operand immIOffset16()
 4624 %{
 4625   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4626   match(ConI);
 4627 
 4628   op_cost(0);
 4629   format %{ %}
 4630   interface(CONST_INTER);
 4631 %}
 4632 
 4633 operand immLoffset()
 4634 %{
 4635   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4636   match(ConL);
 4637 
 4638   op_cost(0);
 4639   format %{ %}
 4640   interface(CONST_INTER);
 4641 %}
 4642 
 4643 operand immLoffset1()
 4644 %{
 4645   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4646   match(ConL);
 4647 
 4648   op_cost(0);
 4649   format %{ %}
 4650   interface(CONST_INTER);
 4651 %}
 4652 
 4653 operand immLoffset2()
 4654 %{
 4655   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4656   match(ConL);
 4657 
 4658   op_cost(0);
 4659   format %{ %}
 4660   interface(CONST_INTER);
 4661 %}
 4662 
 4663 operand immLoffset4()
 4664 %{
 4665   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4666   match(ConL);
 4667 
 4668   op_cost(0);
 4669   format %{ %}
 4670   interface(CONST_INTER);
 4671 %}
 4672 
 4673 operand immLoffset8()
 4674 %{
 4675   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4676   match(ConL);
 4677 
 4678   op_cost(0);
 4679   format %{ %}
 4680   interface(CONST_INTER);
 4681 %}
 4682 
 4683 operand immLoffset16()
 4684 %{
 4685   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4686   match(ConL);
 4687 
 4688   op_cost(0);
 4689   format %{ %}
 4690   interface(CONST_INTER);
 4691 %}
 4692 
 4693 // 8 bit signed value.
 4694 operand immI8()
 4695 %{
 4696   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4697   match(ConI);
 4698 
 4699   op_cost(0);
 4700   format %{ %}
 4701   interface(CONST_INTER);
 4702 %}
 4703 
 4704 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4705 operand immI8_shift8()
 4706 %{
 4707   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4708             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4709   match(ConI);
 4710 
 4711   op_cost(0);
 4712   format %{ %}
 4713   interface(CONST_INTER);
 4714 %}
 4715 
 4716 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4717 operand immL8_shift8()
 4718 %{
 4719   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4720             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4721   match(ConL);
 4722 
 4723   op_cost(0);
 4724   format %{ %}
 4725   interface(CONST_INTER);
 4726 %}
 4727 
 4728 // 8 bit integer valid for vector add sub immediate
 4729 operand immBAddSubV()
 4730 %{
 4731   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4732   match(ConI);
 4733 
 4734   op_cost(0);
 4735   format %{ %}
 4736   interface(CONST_INTER);
 4737 %}
 4738 
 4739 // 32 bit integer valid for add sub immediate
 4740 operand immIAddSub()
 4741 %{
 4742   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4743   match(ConI);
 4744   op_cost(0);
 4745   format %{ %}
 4746   interface(CONST_INTER);
 4747 %}
 4748 
 4749 // 32 bit integer valid for vector add sub immediate
 4750 operand immIAddSubV()
 4751 %{
 4752   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4753   match(ConI);
 4754 
 4755   op_cost(0);
 4756   format %{ %}
 4757   interface(CONST_INTER);
 4758 %}
 4759 
 4760 // 32 bit unsigned integer valid for logical immediate
 4761 
 4762 operand immBLog()
 4763 %{
 4764   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4765   match(ConI);
 4766 
 4767   op_cost(0);
 4768   format %{ %}
 4769   interface(CONST_INTER);
 4770 %}
 4771 
 4772 operand immSLog()
 4773 %{
 4774   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4775   match(ConI);
 4776 
 4777   op_cost(0);
 4778   format %{ %}
 4779   interface(CONST_INTER);
 4780 %}
 4781 
 4782 operand immILog()
 4783 %{
 4784   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4785   match(ConI);
 4786 
 4787   op_cost(0);
 4788   format %{ %}
 4789   interface(CONST_INTER);
 4790 %}
 4791 
 4792 // Integer operands 64 bit
 4793 // 64 bit immediate
 4794 operand immL()
 4795 %{
 4796   match(ConL);
 4797 
 4798   op_cost(0);
 4799   format %{ %}
 4800   interface(CONST_INTER);
 4801 %}
 4802 
 4803 // 64 bit zero
 4804 operand immL0()
 4805 %{
 4806   predicate(n->get_long() == 0);
 4807   match(ConL);
 4808 
 4809   op_cost(0);
 4810   format %{ %}
 4811   interface(CONST_INTER);
 4812 %}
 4813 
 4814 // 64 bit unit increment
 4815 operand immL_1()
 4816 %{
 4817   predicate(n->get_long() == 1);
 4818   match(ConL);
 4819 
 4820   op_cost(0);
 4821   format %{ %}
 4822   interface(CONST_INTER);
 4823 %}
 4824 
 4825 // 64 bit unit decrement
 4826 operand immL_M1()
 4827 %{
 4828   predicate(n->get_long() == -1);
 4829   match(ConL);
 4830 
 4831   op_cost(0);
 4832   format %{ %}
 4833   interface(CONST_INTER);
 4834 %}
 4835 
 4836 // 32 bit offset of pc in thread anchor
 4837 
 4838 operand immL_pc_off()
 4839 %{
 4840   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4841                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4842   match(ConL);
 4843 
 4844   op_cost(0);
 4845   format %{ %}
 4846   interface(CONST_INTER);
 4847 %}
 4848 
 4849 // 64 bit integer valid for add sub immediate
 4850 operand immLAddSub()
 4851 %{
 4852   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4853   match(ConL);
 4854   op_cost(0);
 4855   format %{ %}
 4856   interface(CONST_INTER);
 4857 %}
 4858 
 4859 // 64 bit integer valid for addv subv immediate
 4860 operand immLAddSubV()
 4861 %{
 4862   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4863   match(ConL);
 4864 
 4865   op_cost(0);
 4866   format %{ %}
 4867   interface(CONST_INTER);
 4868 %}
 4869 
 4870 // 64 bit integer valid for logical immediate
 4871 operand immLLog()
 4872 %{
 4873   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4874   match(ConL);
 4875   op_cost(0);
 4876   format %{ %}
 4877   interface(CONST_INTER);
 4878 %}
 4879 
 4880 // Long Immediate: low 32-bit mask
 4881 operand immL_32bits()
 4882 %{
 4883   predicate(n->get_long() == 0xFFFFFFFFL);
 4884   match(ConL);
 4885   op_cost(0);
 4886   format %{ %}
 4887   interface(CONST_INTER);
 4888 %}
 4889 
 4890 // Pointer operands
 4891 // Pointer Immediate
 4892 operand immP()
 4893 %{
 4894   match(ConP);
 4895 
 4896   op_cost(0);
 4897   format %{ %}
 4898   interface(CONST_INTER);
 4899 %}
 4900 
 4901 // NULL Pointer Immediate
 4902 operand immP0()
 4903 %{
 4904   predicate(n->get_ptr() == 0);
 4905   match(ConP);
 4906 
 4907   op_cost(0);
 4908   format %{ %}
 4909   interface(CONST_INTER);
 4910 %}
 4911 
 4912 // Pointer Immediate One
 4913 // this is used in object initialization (initial object header)
 4914 operand immP_1()
 4915 %{
 4916   predicate(n->get_ptr() == 1);
 4917   match(ConP);
 4918 
 4919   op_cost(0);
 4920   format %{ %}
 4921   interface(CONST_INTER);
 4922 %}
 4923 
 4924 // Card Table Byte Map Base
 4925 operand immByteMapBase()
 4926 %{
 4927   // Get base of card map
 4928   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4929             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4930   match(ConP);
 4931 
 4932   op_cost(0);
 4933   format %{ %}
 4934   interface(CONST_INTER);
 4935 %}
 4936 
 4937 // Pointer Immediate Minus One
 4938 // this is used when we want to write the current PC to the thread anchor
 4939 operand immP_M1()
 4940 %{
 4941   predicate(n->get_ptr() == -1);
 4942   match(ConP);
 4943 
 4944   op_cost(0);
 4945   format %{ %}
 4946   interface(CONST_INTER);
 4947 %}
 4948 
 4949 // Pointer Immediate Minus Two
 4950 // this is used when we want to write the current PC to the thread anchor
 4951 operand immP_M2()
 4952 %{
 4953   predicate(n->get_ptr() == -2);
 4954   match(ConP);
 4955 
 4956   op_cost(0);
 4957   format %{ %}
 4958   interface(CONST_INTER);
 4959 %}
 4960 
 4961 // Float and Double operands
 4962 // Double Immediate
 4963 operand immD()
 4964 %{
 4965   match(ConD);
 4966   op_cost(0);
 4967   format %{ %}
 4968   interface(CONST_INTER);
 4969 %}
 4970 
 4971 // Double Immediate: +0.0d
 4972 operand immD0()
 4973 %{
 4974   predicate(jlong_cast(n->getd()) == 0);
 4975   match(ConD);
 4976 
 4977   op_cost(0);
 4978   format %{ %}
 4979   interface(CONST_INTER);
 4980 %}
 4981 
 4982 // constant 'double +0.0'.
 4983 operand immDPacked()
 4984 %{
 4985   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4986   match(ConD);
 4987   op_cost(0);
 4988   format %{ %}
 4989   interface(CONST_INTER);
 4990 %}
 4991 
 4992 // Float Immediate
 4993 operand immF()
 4994 %{
 4995   match(ConF);
 4996   op_cost(0);
 4997   format %{ %}
 4998   interface(CONST_INTER);
 4999 %}
 5000 
 5001 // Float Immediate: +0.0f.
 5002 operand immF0()
 5003 %{
 5004   predicate(jint_cast(n->getf()) == 0);
 5005   match(ConF);
 5006 
 5007   op_cost(0);
 5008   format %{ %}
 5009   interface(CONST_INTER);
 5010 %}
 5011 
 5012 //
 5013 operand immFPacked()
 5014 %{
 5015   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 5016   match(ConF);
 5017   op_cost(0);
 5018   format %{ %}
 5019   interface(CONST_INTER);
 5020 %}
 5021 
 5022 // Narrow pointer operands
 5023 // Narrow Pointer Immediate
 5024 operand immN()
 5025 %{
 5026   match(ConN);
 5027 
 5028   op_cost(0);
 5029   format %{ %}
 5030   interface(CONST_INTER);
 5031 %}
 5032 
 5033 // Narrow NULL Pointer Immediate
 5034 operand immN0()
 5035 %{
 5036   predicate(n->get_narrowcon() == 0);
 5037   match(ConN);
 5038 
 5039   op_cost(0);
 5040   format %{ %}
 5041   interface(CONST_INTER);
 5042 %}
 5043 
 5044 operand immNKlass()
 5045 %{
 5046   match(ConNKlass);
 5047 
 5048   op_cost(0);
 5049   format %{ %}
 5050   interface(CONST_INTER);
 5051 %}
 5052 
 5053 // Integer 32 bit Register Operands
 5054 // Integer 32 bitRegister (excludes SP)
 5055 operand iRegI()
 5056 %{
 5057   constraint(ALLOC_IN_RC(any_reg32));
 5058   match(RegI);
 5059   match(iRegINoSp);
 5060   op_cost(0);
 5061   format %{ %}
 5062   interface(REG_INTER);
 5063 %}
 5064 
 5065 // Integer 32 bit Register not Special
 5066 operand iRegINoSp()
 5067 %{
 5068   constraint(ALLOC_IN_RC(no_special_reg32));
 5069   match(RegI);
 5070   op_cost(0);
 5071   format %{ %}
 5072   interface(REG_INTER);
 5073 %}
 5074 
 5075 // Integer 64 bit Register Operands
 5076 // Integer 64 bit Register (includes SP)
 5077 operand iRegL()
 5078 %{
 5079   constraint(ALLOC_IN_RC(any_reg));
 5080   match(RegL);
 5081   match(iRegLNoSp);
 5082   op_cost(0);
 5083   format %{ %}
 5084   interface(REG_INTER);
 5085 %}
 5086 
 5087 // Integer 64 bit Register not Special
 5088 operand iRegLNoSp()
 5089 %{
 5090   constraint(ALLOC_IN_RC(no_special_reg));
 5091   match(RegL);
 5092   match(iRegL_R0);
 5093   format %{ %}
 5094   interface(REG_INTER);
 5095 %}
 5096 
 5097 // Pointer Register Operands
 5098 // Pointer Register
 5099 operand iRegP()
 5100 %{
 5101   constraint(ALLOC_IN_RC(ptr_reg));
 5102   match(RegP);
 5103   match(iRegPNoSp);
 5104   match(iRegP_R0);
 5105   //match(iRegP_R2);
 5106   //match(iRegP_R4);
 5107   //match(iRegP_R5);
 5108   match(thread_RegP);
 5109   op_cost(0);
 5110   format %{ %}
 5111   interface(REG_INTER);
 5112 %}
 5113 
 5114 // Pointer 64 bit Register not Special
 5115 operand iRegPNoSp()
 5116 %{
 5117   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 5118   match(RegP);
 5119   // match(iRegP);
 5120   // match(iRegP_R0);
 5121   // match(iRegP_R2);
 5122   // match(iRegP_R4);
 5123   // match(iRegP_R5);
 5124   // match(thread_RegP);
 5125   op_cost(0);
 5126   format %{ %}
 5127   interface(REG_INTER);
 5128 %}
 5129 
 5130 // Pointer 64 bit Register R0 only
 5131 operand iRegP_R0()
 5132 %{
 5133   constraint(ALLOC_IN_RC(r0_reg));
 5134   match(RegP);
 5135   // match(iRegP);
 5136   match(iRegPNoSp);
 5137   op_cost(0);
 5138   format %{ %}
 5139   interface(REG_INTER);
 5140 %}
 5141 
 5142 // Pointer 64 bit Register R1 only
 5143 operand iRegP_R1()
 5144 %{
 5145   constraint(ALLOC_IN_RC(r1_reg));
 5146   match(RegP);
 5147   // match(iRegP);
 5148   match(iRegPNoSp);
 5149   op_cost(0);
 5150   format %{ %}
 5151   interface(REG_INTER);
 5152 %}
 5153 
 5154 // Pointer 64 bit Register R2 only
 5155 operand iRegP_R2()
 5156 %{
 5157   constraint(ALLOC_IN_RC(r2_reg));
 5158   match(RegP);
 5159   // match(iRegP);
 5160   match(iRegPNoSp);
 5161   op_cost(0);
 5162   format %{ %}
 5163   interface(REG_INTER);
 5164 %}
 5165 
 5166 // Pointer 64 bit Register R3 only
 5167 operand iRegP_R3()
 5168 %{
 5169   constraint(ALLOC_IN_RC(r3_reg));
 5170   match(RegP);
 5171   // match(iRegP);
 5172   match(iRegPNoSp);
 5173   op_cost(0);
 5174   format %{ %}
 5175   interface(REG_INTER);
 5176 %}
 5177 
 5178 // Pointer 64 bit Register R4 only
 5179 operand iRegP_R4()
 5180 %{
 5181   constraint(ALLOC_IN_RC(r4_reg));
 5182   match(RegP);
 5183   // match(iRegP);
 5184   match(iRegPNoSp);
 5185   op_cost(0);
 5186   format %{ %}
 5187   interface(REG_INTER);
 5188 %}
 5189 
 5190 // Pointer 64 bit Register R5 only
 5191 operand iRegP_R5()
 5192 %{
 5193   constraint(ALLOC_IN_RC(r5_reg));
 5194   match(RegP);
 5195   // match(iRegP);
 5196   match(iRegPNoSp);
 5197   op_cost(0);
 5198   format %{ %}
 5199   interface(REG_INTER);
 5200 %}
 5201 
 5202 // Pointer 64 bit Register R10 only
 5203 operand iRegP_R10()
 5204 %{
 5205   constraint(ALLOC_IN_RC(r10_reg));
 5206   match(RegP);
 5207   // match(iRegP);
 5208   match(iRegPNoSp);
 5209   op_cost(0);
 5210   format %{ %}
 5211   interface(REG_INTER);
 5212 %}
 5213 
 5214 // Long 64 bit Register R0 only
 5215 operand iRegL_R0()
 5216 %{
 5217   constraint(ALLOC_IN_RC(r0_reg));
 5218   match(RegL);
 5219   match(iRegLNoSp);
 5220   op_cost(0);
 5221   format %{ %}
 5222   interface(REG_INTER);
 5223 %}
 5224 
 5225 // Long 64 bit Register R2 only
 5226 operand iRegL_R2()
 5227 %{
 5228   constraint(ALLOC_IN_RC(r2_reg));
 5229   match(RegL);
 5230   match(iRegLNoSp);
 5231   op_cost(0);
 5232   format %{ %}
 5233   interface(REG_INTER);
 5234 %}
 5235 
 5236 // Long 64 bit Register R3 only
 5237 operand iRegL_R3()
 5238 %{
 5239   constraint(ALLOC_IN_RC(r3_reg));
 5240   match(RegL);
 5241   match(iRegLNoSp);
 5242   op_cost(0);
 5243   format %{ %}
 5244   interface(REG_INTER);
 5245 %}
 5246 
 5247 // Long 64 bit Register R11 only
 5248 operand iRegL_R11()
 5249 %{
 5250   constraint(ALLOC_IN_RC(r11_reg));
 5251   match(RegL);
 5252   match(iRegLNoSp);
 5253   op_cost(0);
 5254   format %{ %}
 5255   interface(REG_INTER);
 5256 %}
 5257 
 5258 // Pointer 64 bit Register FP only
 5259 operand iRegP_FP()
 5260 %{
 5261   constraint(ALLOC_IN_RC(fp_reg));
 5262   match(RegP);
 5263   // match(iRegP);
 5264   op_cost(0);
 5265   format %{ %}
 5266   interface(REG_INTER);
 5267 %}
 5268 
 5269 // Register R0 only
 5270 operand iRegI_R0()
 5271 %{
 5272   constraint(ALLOC_IN_RC(int_r0_reg));
 5273   match(RegI);
 5274   match(iRegINoSp);
 5275   op_cost(0);
 5276   format %{ %}
 5277   interface(REG_INTER);
 5278 %}
 5279 
 5280 // Register R2 only
 5281 operand iRegI_R2()
 5282 %{
 5283   constraint(ALLOC_IN_RC(int_r2_reg));
 5284   match(RegI);
 5285   match(iRegINoSp);
 5286   op_cost(0);
 5287   format %{ %}
 5288   interface(REG_INTER);
 5289 %}
 5290 
 5291 // Register R3 only
 5292 operand iRegI_R3()
 5293 %{
 5294   constraint(ALLOC_IN_RC(int_r3_reg));
 5295   match(RegI);
 5296   match(iRegINoSp);
 5297   op_cost(0);
 5298   format %{ %}
 5299   interface(REG_INTER);
 5300 %}
 5301 
 5302 
 5303 // Register R4 only
 5304 operand iRegI_R4()
 5305 %{
 5306   constraint(ALLOC_IN_RC(int_r4_reg));
 5307   match(RegI);
 5308   match(iRegINoSp);
 5309   op_cost(0);
 5310   format %{ %}
 5311   interface(REG_INTER);
 5312 %}
 5313 
 5314 
 5315 // Pointer Register Operands
 5316 // Narrow Pointer Register
 5317 operand iRegN()
 5318 %{
 5319   constraint(ALLOC_IN_RC(any_reg32));
 5320   match(RegN);
 5321   match(iRegNNoSp);
 5322   op_cost(0);
 5323   format %{ %}
 5324   interface(REG_INTER);
 5325 %}
 5326 
 5327 operand iRegN_R0()
 5328 %{
 5329   constraint(ALLOC_IN_RC(r0_reg));
 5330   match(iRegN);
 5331   op_cost(0);
 5332   format %{ %}
 5333   interface(REG_INTER);
 5334 %}
 5335 
 5336 operand iRegN_R2()
 5337 %{
 5338   constraint(ALLOC_IN_RC(r2_reg));
 5339   match(iRegN);
 5340   op_cost(0);
 5341   format %{ %}
 5342   interface(REG_INTER);
 5343 %}
 5344 
 5345 operand iRegN_R3()
 5346 %{
 5347   constraint(ALLOC_IN_RC(r3_reg));
 5348   match(iRegN);
 5349   op_cost(0);
 5350   format %{ %}
 5351   interface(REG_INTER);
 5352 %}
 5353 
 5354 // Integer 64 bit Register not Special
 5355 operand iRegNNoSp()
 5356 %{
 5357   constraint(ALLOC_IN_RC(no_special_reg32));
 5358   match(RegN);
 5359   op_cost(0);
 5360   format %{ %}
 5361   interface(REG_INTER);
 5362 %}
 5363 
 5364 // heap base register -- used for encoding immN0
 5365 
 5366 operand iRegIHeapbase()
 5367 %{
 5368   constraint(ALLOC_IN_RC(heapbase_reg));
 5369   match(RegI);
 5370   op_cost(0);
 5371   format %{ %}
 5372   interface(REG_INTER);
 5373 %}
 5374 
 5375 // Float Register
 5376 // Float register operands
 5377 operand vRegF()
 5378 %{
 5379   constraint(ALLOC_IN_RC(float_reg));
 5380   match(RegF);
 5381 
 5382   op_cost(0);
 5383   format %{ %}
 5384   interface(REG_INTER);
 5385 %}
 5386 
 5387 // Double Register
 5388 // Double register operands
 5389 operand vRegD()
 5390 %{
 5391   constraint(ALLOC_IN_RC(double_reg));
 5392   match(RegD);
 5393 
 5394   op_cost(0);
 5395   format %{ %}
 5396   interface(REG_INTER);
 5397 %}
 5398 
 5399 // Generic vector class. This will be used for
 5400 // all vector operands, including NEON and SVE,
 5401 // but currently only used for SVE VecA.
 5402 operand vReg()
 5403 %{
 5404   constraint(ALLOC_IN_RC(vectora_reg));
 5405   match(VecA);
 5406   op_cost(0);
 5407   format %{ %}
 5408   interface(REG_INTER);
 5409 %}
 5410 
 5411 operand vecD()
 5412 %{
 5413   constraint(ALLOC_IN_RC(vectord_reg));
 5414   match(VecD);
 5415 
 5416   op_cost(0);
 5417   format %{ %}
 5418   interface(REG_INTER);
 5419 %}
 5420 
 5421 operand vecX()
 5422 %{
 5423   constraint(ALLOC_IN_RC(vectorx_reg));
 5424   match(VecX);
 5425 
 5426   op_cost(0);
 5427   format %{ %}
 5428   interface(REG_INTER);
 5429 %}
 5430 
 5431 operand vRegD_V0()
 5432 %{
 5433   constraint(ALLOC_IN_RC(v0_reg));
 5434   match(RegD);
 5435   op_cost(0);
 5436   format %{ %}
 5437   interface(REG_INTER);
 5438 %}
 5439 
 5440 operand vRegD_V1()
 5441 %{
 5442   constraint(ALLOC_IN_RC(v1_reg));
 5443   match(RegD);
 5444   op_cost(0);
 5445   format %{ %}
 5446   interface(REG_INTER);
 5447 %}
 5448 
 5449 operand vRegD_V2()
 5450 %{
 5451   constraint(ALLOC_IN_RC(v2_reg));
 5452   match(RegD);
 5453   op_cost(0);
 5454   format %{ %}
 5455   interface(REG_INTER);
 5456 %}
 5457 
 5458 operand vRegD_V3()
 5459 %{
 5460   constraint(ALLOC_IN_RC(v3_reg));
 5461   match(RegD);
 5462   op_cost(0);
 5463   format %{ %}
 5464   interface(REG_INTER);
 5465 %}
 5466 
 5467 operand vRegD_V4()
 5468 %{
 5469   constraint(ALLOC_IN_RC(v4_reg));
 5470   match(RegD);
 5471   op_cost(0);
 5472   format %{ %}
 5473   interface(REG_INTER);
 5474 %}
 5475 
 5476 operand vRegD_V5()
 5477 %{
 5478   constraint(ALLOC_IN_RC(v5_reg));
 5479   match(RegD);
 5480   op_cost(0);
 5481   format %{ %}
 5482   interface(REG_INTER);
 5483 %}
 5484 
 5485 operand vRegD_V6()
 5486 %{
 5487   constraint(ALLOC_IN_RC(v6_reg));
 5488   match(RegD);
 5489   op_cost(0);
 5490   format %{ %}
 5491   interface(REG_INTER);
 5492 %}
 5493 
 5494 operand vRegD_V7()
 5495 %{
 5496   constraint(ALLOC_IN_RC(v7_reg));
 5497   match(RegD);
 5498   op_cost(0);
 5499   format %{ %}
 5500   interface(REG_INTER);
 5501 %}
 5502 
 5503 operand vRegD_V8()
 5504 %{
 5505   constraint(ALLOC_IN_RC(v8_reg));
 5506   match(RegD);
 5507   op_cost(0);
 5508   format %{ %}
 5509   interface(REG_INTER);
 5510 %}
 5511 
 5512 operand vRegD_V9()
 5513 %{
 5514   constraint(ALLOC_IN_RC(v9_reg));
 5515   match(RegD);
 5516   op_cost(0);
 5517   format %{ %}
 5518   interface(REG_INTER);
 5519 %}
 5520 
 5521 operand vRegD_V10()
 5522 %{
 5523   constraint(ALLOC_IN_RC(v10_reg));
 5524   match(RegD);
 5525   op_cost(0);
 5526   format %{ %}
 5527   interface(REG_INTER);
 5528 %}
 5529 
 5530 operand vRegD_V11()
 5531 %{
 5532   constraint(ALLOC_IN_RC(v11_reg));
 5533   match(RegD);
 5534   op_cost(0);
 5535   format %{ %}
 5536   interface(REG_INTER);
 5537 %}
 5538 
 5539 operand vRegD_V12()
 5540 %{
 5541   constraint(ALLOC_IN_RC(v12_reg));
 5542   match(RegD);
 5543   op_cost(0);
 5544   format %{ %}
 5545   interface(REG_INTER);
 5546 %}
 5547 
 5548 operand vRegD_V13()
 5549 %{
 5550   constraint(ALLOC_IN_RC(v13_reg));
 5551   match(RegD);
 5552   op_cost(0);
 5553   format %{ %}
 5554   interface(REG_INTER);
 5555 %}
 5556 
 5557 operand vRegD_V14()
 5558 %{
 5559   constraint(ALLOC_IN_RC(v14_reg));
 5560   match(RegD);
 5561   op_cost(0);
 5562   format %{ %}
 5563   interface(REG_INTER);
 5564 %}
 5565 
 5566 operand vRegD_V15()
 5567 %{
 5568   constraint(ALLOC_IN_RC(v15_reg));
 5569   match(RegD);
 5570   op_cost(0);
 5571   format %{ %}
 5572   interface(REG_INTER);
 5573 %}
 5574 
 5575 operand vRegD_V16()
 5576 %{
 5577   constraint(ALLOC_IN_RC(v16_reg));
 5578   match(RegD);
 5579   op_cost(0);
 5580   format %{ %}
 5581   interface(REG_INTER);
 5582 %}
 5583 
 5584 operand vRegD_V17()
 5585 %{
 5586   constraint(ALLOC_IN_RC(v17_reg));
 5587   match(RegD);
 5588   op_cost(0);
 5589   format %{ %}
 5590   interface(REG_INTER);
 5591 %}
 5592 
 5593 operand vRegD_V18()
 5594 %{
 5595   constraint(ALLOC_IN_RC(v18_reg));
 5596   match(RegD);
 5597   op_cost(0);
 5598   format %{ %}
 5599   interface(REG_INTER);
 5600 %}
 5601 
 5602 operand vRegD_V19()
 5603 %{
 5604   constraint(ALLOC_IN_RC(v19_reg));
 5605   match(RegD);
 5606   op_cost(0);
 5607   format %{ %}
 5608   interface(REG_INTER);
 5609 %}
 5610 
 5611 operand vRegD_V20()
 5612 %{
 5613   constraint(ALLOC_IN_RC(v20_reg));
 5614   match(RegD);
 5615   op_cost(0);
 5616   format %{ %}
 5617   interface(REG_INTER);
 5618 %}
 5619 
 5620 operand vRegD_V21()
 5621 %{
 5622   constraint(ALLOC_IN_RC(v21_reg));
 5623   match(RegD);
 5624   op_cost(0);
 5625   format %{ %}
 5626   interface(REG_INTER);
 5627 %}
 5628 
 5629 operand vRegD_V22()
 5630 %{
 5631   constraint(ALLOC_IN_RC(v22_reg));
 5632   match(RegD);
 5633   op_cost(0);
 5634   format %{ %}
 5635   interface(REG_INTER);
 5636 %}
 5637 
 5638 operand vRegD_V23()
 5639 %{
 5640   constraint(ALLOC_IN_RC(v23_reg));
 5641   match(RegD);
 5642   op_cost(0);
 5643   format %{ %}
 5644   interface(REG_INTER);
 5645 %}
 5646 
 5647 operand vRegD_V24()
 5648 %{
 5649   constraint(ALLOC_IN_RC(v24_reg));
 5650   match(RegD);
 5651   op_cost(0);
 5652   format %{ %}
 5653   interface(REG_INTER);
 5654 %}
 5655 
 5656 operand vRegD_V25()
 5657 %{
 5658   constraint(ALLOC_IN_RC(v25_reg));
 5659   match(RegD);
 5660   op_cost(0);
 5661   format %{ %}
 5662   interface(REG_INTER);
 5663 %}
 5664 
 5665 operand vRegD_V26()
 5666 %{
 5667   constraint(ALLOC_IN_RC(v26_reg));
 5668   match(RegD);
 5669   op_cost(0);
 5670   format %{ %}
 5671   interface(REG_INTER);
 5672 %}
 5673 
 5674 operand vRegD_V27()
 5675 %{
 5676   constraint(ALLOC_IN_RC(v27_reg));
 5677   match(RegD);
 5678   op_cost(0);
 5679   format %{ %}
 5680   interface(REG_INTER);
 5681 %}
 5682 
 5683 operand vRegD_V28()
 5684 %{
 5685   constraint(ALLOC_IN_RC(v28_reg));
 5686   match(RegD);
 5687   op_cost(0);
 5688   format %{ %}
 5689   interface(REG_INTER);
 5690 %}
 5691 
 5692 operand vRegD_V29()
 5693 %{
 5694   constraint(ALLOC_IN_RC(v29_reg));
 5695   match(RegD);
 5696   op_cost(0);
 5697   format %{ %}
 5698   interface(REG_INTER);
 5699 %}
 5700 
 5701 operand vRegD_V30()
 5702 %{
 5703   constraint(ALLOC_IN_RC(v30_reg));
 5704   match(RegD);
 5705   op_cost(0);
 5706   format %{ %}
 5707   interface(REG_INTER);
 5708 %}
 5709 
 5710 operand vRegD_V31()
 5711 %{
 5712   constraint(ALLOC_IN_RC(v31_reg));
 5713   match(RegD);
 5714   op_cost(0);
 5715   format %{ %}
 5716   interface(REG_INTER);
 5717 %}
 5718 
 5719 operand pReg()
 5720 %{
 5721   constraint(ALLOC_IN_RC(pr_reg));
 5722   match(RegVectMask);
 5723   match(pRegGov);
 5724   op_cost(0);
 5725   format %{ %}
 5726   interface(REG_INTER);
 5727 %}
 5728 
 5729 operand pRegGov()
 5730 %{
 5731   constraint(ALLOC_IN_RC(gov_pr));
 5732   match(RegVectMask);
 5733   op_cost(0);
 5734   format %{ %}
 5735   interface(REG_INTER);
 5736 %}
 5737 
 5738 // Flags register, used as output of signed compare instructions
 5739 
 5740 // note that on AArch64 we also use this register as the output for
 5741 // for floating point compare instructions (CmpF CmpD). this ensures
 5742 // that ordered inequality tests use GT, GE, LT or LE none of which
 5743 // pass through cases where the result is unordered i.e. one or both
 5744 // inputs to the compare is a NaN. this means that the ideal code can
 5745 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5746 // (where the comparison should always fail). EQ and NE tests are
 5747 // always generated in ideal code so that unordered folds into the NE
 5748 // case, matching the behaviour of AArch64 NE.
 5749 //
 5750 // This differs from x86 where the outputs of FP compares use a
 5751 // special FP flags registers and where compares based on this
 5752 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5753 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5754 // to explicitly handle the unordered case in branches. x86 also has
 5755 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5756 
 5757 operand rFlagsReg()
 5758 %{
 5759   constraint(ALLOC_IN_RC(int_flags));
 5760   match(RegFlags);
 5761 
 5762   op_cost(0);
 5763   format %{ "RFLAGS" %}
 5764   interface(REG_INTER);
 5765 %}
 5766 
 5767 // Flags register, used as output of unsigned compare instructions
 5768 operand rFlagsRegU()
 5769 %{
 5770   constraint(ALLOC_IN_RC(int_flags));
 5771   match(RegFlags);
 5772 
 5773   op_cost(0);
 5774   format %{ "RFLAGSU" %}
 5775   interface(REG_INTER);
 5776 %}
 5777 
 5778 // Special Registers
 5779 
 5780 // Method Register
 5781 operand inline_cache_RegP(iRegP reg)
 5782 %{
 5783   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5784   match(reg);
 5785   match(iRegPNoSp);
 5786   op_cost(0);
 5787   format %{ %}
 5788   interface(REG_INTER);
 5789 %}
 5790 
 5791 // Thread Register
 5792 operand thread_RegP(iRegP reg)
 5793 %{
 5794   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5795   match(reg);
 5796   op_cost(0);
 5797   format %{ %}
 5798   interface(REG_INTER);
 5799 %}
 5800 
 5801 operand lr_RegP(iRegP reg)
 5802 %{
 5803   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5804   match(reg);
 5805   op_cost(0);
 5806   format %{ %}
 5807   interface(REG_INTER);
 5808 %}
 5809 
 5810 //----------Memory Operands----------------------------------------------------
 5811 
 5812 operand indirect(iRegP reg)
 5813 %{
 5814   constraint(ALLOC_IN_RC(ptr_reg));
 5815   match(reg);
 5816   op_cost(0);
 5817   format %{ "[$reg]" %}
 5818   interface(MEMORY_INTER) %{
 5819     base($reg);
 5820     index(0xffffffff);
 5821     scale(0x0);
 5822     disp(0x0);
 5823   %}
 5824 %}
 5825 
 5826 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5827 %{
 5828   constraint(ALLOC_IN_RC(ptr_reg));
 5829   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5830   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5831   op_cost(0);
 5832   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5833   interface(MEMORY_INTER) %{
 5834     base($reg);
 5835     index($ireg);
 5836     scale($scale);
 5837     disp(0x0);
 5838   %}
 5839 %}
 5840 
 5841 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5842 %{
 5843   constraint(ALLOC_IN_RC(ptr_reg));
 5844   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5845   match(AddP reg (LShiftL lreg scale));
 5846   op_cost(0);
 5847   format %{ "$reg, $lreg lsl($scale)" %}
 5848   interface(MEMORY_INTER) %{
 5849     base($reg);
 5850     index($lreg);
 5851     scale($scale);
 5852     disp(0x0);
 5853   %}
 5854 %}
 5855 
 5856 operand indIndexI2L(iRegP reg, iRegI ireg)
 5857 %{
 5858   constraint(ALLOC_IN_RC(ptr_reg));
 5859   match(AddP reg (ConvI2L ireg));
 5860   op_cost(0);
 5861   format %{ "$reg, $ireg, 0, I2L" %}
 5862   interface(MEMORY_INTER) %{
 5863     base($reg);
 5864     index($ireg);
 5865     scale(0x0);
 5866     disp(0x0);
 5867   %}
 5868 %}
 5869 
 5870 operand indIndex(iRegP reg, iRegL lreg)
 5871 %{
 5872   constraint(ALLOC_IN_RC(ptr_reg));
 5873   match(AddP reg lreg);
 5874   op_cost(0);
 5875   format %{ "$reg, $lreg" %}
 5876   interface(MEMORY_INTER) %{
 5877     base($reg);
 5878     index($lreg);
 5879     scale(0x0);
 5880     disp(0x0);
 5881   %}
 5882 %}
 5883 
 5884 operand indOffI(iRegP reg, immIOffset off)
 5885 %{
 5886   constraint(ALLOC_IN_RC(ptr_reg));
 5887   match(AddP reg off);
 5888   op_cost(0);
 5889   format %{ "[$reg, $off]" %}
 5890   interface(MEMORY_INTER) %{
 5891     base($reg);
 5892     index(0xffffffff);
 5893     scale(0x0);
 5894     disp($off);
 5895   %}
 5896 %}
 5897 
 5898 operand indOffI1(iRegP reg, immIOffset1 off)
 5899 %{
 5900   constraint(ALLOC_IN_RC(ptr_reg));
 5901   match(AddP reg off);
 5902   op_cost(0);
 5903   format %{ "[$reg, $off]" %}
 5904   interface(MEMORY_INTER) %{
 5905     base($reg);
 5906     index(0xffffffff);
 5907     scale(0x0);
 5908     disp($off);
 5909   %}
 5910 %}
 5911 
 5912 operand indOffI2(iRegP reg, immIOffset2 off)
 5913 %{
 5914   constraint(ALLOC_IN_RC(ptr_reg));
 5915   match(AddP reg off);
 5916   op_cost(0);
 5917   format %{ "[$reg, $off]" %}
 5918   interface(MEMORY_INTER) %{
 5919     base($reg);
 5920     index(0xffffffff);
 5921     scale(0x0);
 5922     disp($off);
 5923   %}
 5924 %}
 5925 
 5926 operand indOffI4(iRegP reg, immIOffset4 off)
 5927 %{
 5928   constraint(ALLOC_IN_RC(ptr_reg));
 5929   match(AddP reg off);
 5930   op_cost(0);
 5931   format %{ "[$reg, $off]" %}
 5932   interface(MEMORY_INTER) %{
 5933     base($reg);
 5934     index(0xffffffff);
 5935     scale(0x0);
 5936     disp($off);
 5937   %}
 5938 %}
 5939 
 5940 operand indOffI8(iRegP reg, immIOffset8 off)
 5941 %{
 5942   constraint(ALLOC_IN_RC(ptr_reg));
 5943   match(AddP reg off);
 5944   op_cost(0);
 5945   format %{ "[$reg, $off]" %}
 5946   interface(MEMORY_INTER) %{
 5947     base($reg);
 5948     index(0xffffffff);
 5949     scale(0x0);
 5950     disp($off);
 5951   %}
 5952 %}
 5953 
 5954 operand indOffI16(iRegP reg, immIOffset16 off)
 5955 %{
 5956   constraint(ALLOC_IN_RC(ptr_reg));
 5957   match(AddP reg off);
 5958   op_cost(0);
 5959   format %{ "[$reg, $off]" %}
 5960   interface(MEMORY_INTER) %{
 5961     base($reg);
 5962     index(0xffffffff);
 5963     scale(0x0);
 5964     disp($off);
 5965   %}
 5966 %}
 5967 
 5968 operand indOffL(iRegP reg, immLoffset off)
 5969 %{
 5970   constraint(ALLOC_IN_RC(ptr_reg));
 5971   match(AddP reg off);
 5972   op_cost(0);
 5973   format %{ "[$reg, $off]" %}
 5974   interface(MEMORY_INTER) %{
 5975     base($reg);
 5976     index(0xffffffff);
 5977     scale(0x0);
 5978     disp($off);
 5979   %}
 5980 %}
 5981 
 5982 operand indOffL1(iRegP reg, immLoffset1 off)
 5983 %{
 5984   constraint(ALLOC_IN_RC(ptr_reg));
 5985   match(AddP reg off);
 5986   op_cost(0);
 5987   format %{ "[$reg, $off]" %}
 5988   interface(MEMORY_INTER) %{
 5989     base($reg);
 5990     index(0xffffffff);
 5991     scale(0x0);
 5992     disp($off);
 5993   %}
 5994 %}
 5995 
 5996 operand indOffL2(iRegP reg, immLoffset2 off)
 5997 %{
 5998   constraint(ALLOC_IN_RC(ptr_reg));
 5999   match(AddP reg off);
 6000   op_cost(0);
 6001   format %{ "[$reg, $off]" %}
 6002   interface(MEMORY_INTER) %{
 6003     base($reg);
 6004     index(0xffffffff);
 6005     scale(0x0);
 6006     disp($off);
 6007   %}
 6008 %}
 6009 
 6010 operand indOffL4(iRegP reg, immLoffset4 off)
 6011 %{
 6012   constraint(ALLOC_IN_RC(ptr_reg));
 6013   match(AddP reg off);
 6014   op_cost(0);
 6015   format %{ "[$reg, $off]" %}
 6016   interface(MEMORY_INTER) %{
 6017     base($reg);
 6018     index(0xffffffff);
 6019     scale(0x0);
 6020     disp($off);
 6021   %}
 6022 %}
 6023 
 6024 operand indOffL8(iRegP reg, immLoffset8 off)
 6025 %{
 6026   constraint(ALLOC_IN_RC(ptr_reg));
 6027   match(AddP reg off);
 6028   op_cost(0);
 6029   format %{ "[$reg, $off]" %}
 6030   interface(MEMORY_INTER) %{
 6031     base($reg);
 6032     index(0xffffffff);
 6033     scale(0x0);
 6034     disp($off);
 6035   %}
 6036 %}
 6037 
 6038 operand indOffL16(iRegP reg, immLoffset16 off)
 6039 %{
 6040   constraint(ALLOC_IN_RC(ptr_reg));
 6041   match(AddP reg off);
 6042   op_cost(0);
 6043   format %{ "[$reg, $off]" %}
 6044   interface(MEMORY_INTER) %{
 6045     base($reg);
 6046     index(0xffffffff);
 6047     scale(0x0);
 6048     disp($off);
 6049   %}
 6050 %}
 6051 
 6052 operand indirectN(iRegN reg)
 6053 %{
 6054   predicate(CompressedOops::shift() == 0);
 6055   constraint(ALLOC_IN_RC(ptr_reg));
 6056   match(DecodeN reg);
 6057   op_cost(0);
 6058   format %{ "[$reg]\t# narrow" %}
 6059   interface(MEMORY_INTER) %{
 6060     base($reg);
 6061     index(0xffffffff);
 6062     scale(0x0);
 6063     disp(0x0);
 6064   %}
 6065 %}
 6066 
 6067 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 6068 %{
 6069   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6070   constraint(ALLOC_IN_RC(ptr_reg));
 6071   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 6072   op_cost(0);
 6073   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 6074   interface(MEMORY_INTER) %{
 6075     base($reg);
 6076     index($ireg);
 6077     scale($scale);
 6078     disp(0x0);
 6079   %}
 6080 %}
 6081 
 6082 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 6083 %{
 6084   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6085   constraint(ALLOC_IN_RC(ptr_reg));
 6086   match(AddP (DecodeN reg) (LShiftL lreg scale));
 6087   op_cost(0);
 6088   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 6089   interface(MEMORY_INTER) %{
 6090     base($reg);
 6091     index($lreg);
 6092     scale($scale);
 6093     disp(0x0);
 6094   %}
 6095 %}
 6096 
 6097 operand indIndexI2LN(iRegN reg, iRegI ireg)
 6098 %{
 6099   predicate(CompressedOops::shift() == 0);
 6100   constraint(ALLOC_IN_RC(ptr_reg));
 6101   match(AddP (DecodeN reg) (ConvI2L ireg));
 6102   op_cost(0);
 6103   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 6104   interface(MEMORY_INTER) %{
 6105     base($reg);
 6106     index($ireg);
 6107     scale(0x0);
 6108     disp(0x0);
 6109   %}
 6110 %}
 6111 
 6112 operand indIndexN(iRegN reg, iRegL lreg)
 6113 %{
 6114   predicate(CompressedOops::shift() == 0);
 6115   constraint(ALLOC_IN_RC(ptr_reg));
 6116   match(AddP (DecodeN reg) lreg);
 6117   op_cost(0);
 6118   format %{ "$reg, $lreg\t# narrow" %}
 6119   interface(MEMORY_INTER) %{
 6120     base($reg);
 6121     index($lreg);
 6122     scale(0x0);
 6123     disp(0x0);
 6124   %}
 6125 %}
 6126 
 6127 operand indOffIN(iRegN reg, immIOffset off)
 6128 %{
 6129   predicate(CompressedOops::shift() == 0);
 6130   constraint(ALLOC_IN_RC(ptr_reg));
 6131   match(AddP (DecodeN reg) off);
 6132   op_cost(0);
 6133   format %{ "[$reg, $off]\t# narrow" %}
 6134   interface(MEMORY_INTER) %{
 6135     base($reg);
 6136     index(0xffffffff);
 6137     scale(0x0);
 6138     disp($off);
 6139   %}
 6140 %}
 6141 
 6142 operand indOffLN(iRegN reg, immLoffset off)
 6143 %{
 6144   predicate(CompressedOops::shift() == 0);
 6145   constraint(ALLOC_IN_RC(ptr_reg));
 6146   match(AddP (DecodeN reg) off);
 6147   op_cost(0);
 6148   format %{ "[$reg, $off]\t# narrow" %}
 6149   interface(MEMORY_INTER) %{
 6150     base($reg);
 6151     index(0xffffffff);
 6152     scale(0x0);
 6153     disp($off);
 6154   %}
 6155 %}
 6156 
 6157 
 6158 
 6159 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6160 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6161 %{
 6162   constraint(ALLOC_IN_RC(ptr_reg));
 6163   match(AddP reg off);
 6164   op_cost(0);
 6165   format %{ "[$reg, $off]" %}
 6166   interface(MEMORY_INTER) %{
 6167     base($reg);
 6168     index(0xffffffff);
 6169     scale(0x0);
 6170     disp($off);
 6171   %}
 6172 %}
 6173 
 6174 //----------Special Memory Operands--------------------------------------------
 6175 // Stack Slot Operand - This operand is used for loading and storing temporary
 6176 //                      values on the stack where a match requires a value to
 6177 //                      flow through memory.
 6178 operand stackSlotP(sRegP reg)
 6179 %{
 6180   constraint(ALLOC_IN_RC(stack_slots));
 6181   op_cost(100);
 6182   // No match rule because this operand is only generated in matching
 6183   // match(RegP);
 6184   format %{ "[$reg]" %}
 6185   interface(MEMORY_INTER) %{
 6186     base(0x1e);  // RSP
 6187     index(0x0);  // No Index
 6188     scale(0x0);  // No Scale
 6189     disp($reg);  // Stack Offset
 6190   %}
 6191 %}
 6192 
 6193 operand stackSlotI(sRegI reg)
 6194 %{
 6195   constraint(ALLOC_IN_RC(stack_slots));
 6196   // No match rule because this operand is only generated in matching
 6197   // match(RegI);
 6198   format %{ "[$reg]" %}
 6199   interface(MEMORY_INTER) %{
 6200     base(0x1e);  // RSP
 6201     index(0x0);  // No Index
 6202     scale(0x0);  // No Scale
 6203     disp($reg);  // Stack Offset
 6204   %}
 6205 %}
 6206 
 6207 operand stackSlotF(sRegF reg)
 6208 %{
 6209   constraint(ALLOC_IN_RC(stack_slots));
 6210   // No match rule because this operand is only generated in matching
 6211   // match(RegF);
 6212   format %{ "[$reg]" %}
 6213   interface(MEMORY_INTER) %{
 6214     base(0x1e);  // RSP
 6215     index(0x0);  // No Index
 6216     scale(0x0);  // No Scale
 6217     disp($reg);  // Stack Offset
 6218   %}
 6219 %}
 6220 
 6221 operand stackSlotD(sRegD reg)
 6222 %{
 6223   constraint(ALLOC_IN_RC(stack_slots));
 6224   // No match rule because this operand is only generated in matching
 6225   // match(RegD);
 6226   format %{ "[$reg]" %}
 6227   interface(MEMORY_INTER) %{
 6228     base(0x1e);  // RSP
 6229     index(0x0);  // No Index
 6230     scale(0x0);  // No Scale
 6231     disp($reg);  // Stack Offset
 6232   %}
 6233 %}
 6234 
 6235 operand stackSlotL(sRegL reg)
 6236 %{
 6237   constraint(ALLOC_IN_RC(stack_slots));
 6238   // No match rule because this operand is only generated in matching
 6239   // match(RegL);
 6240   format %{ "[$reg]" %}
 6241   interface(MEMORY_INTER) %{
 6242     base(0x1e);  // RSP
 6243     index(0x0);  // No Index
 6244     scale(0x0);  // No Scale
 6245     disp($reg);  // Stack Offset
 6246   %}
 6247 %}
 6248 
 6249 // Operands for expressing Control Flow
 6250 // NOTE: Label is a predefined operand which should not be redefined in
 6251 //       the AD file. It is generically handled within the ADLC.
 6252 
 6253 //----------Conditional Branch Operands----------------------------------------
 6254 // Comparison Op  - This is the operation of the comparison, and is limited to
 6255 //                  the following set of codes:
 6256 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6257 //
 6258 // Other attributes of the comparison, such as unsignedness, are specified
 6259 // by the comparison instruction that sets a condition code flags register.
 6260 // That result is represented by a flags operand whose subtype is appropriate
 6261 // to the unsignedness (etc.) of the comparison.
 6262 //
 6263 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6264 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6265 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6266 
 6267 // used for signed integral comparisons and fp comparisons
 6268 
 6269 operand cmpOp()
 6270 %{
 6271   match(Bool);
 6272 
 6273   format %{ "" %}
 6274   interface(COND_INTER) %{
 6275     equal(0x0, "eq");
 6276     not_equal(0x1, "ne");
 6277     less(0xb, "lt");
 6278     greater_equal(0xa, "ge");
 6279     less_equal(0xd, "le");
 6280     greater(0xc, "gt");
 6281     overflow(0x6, "vs");
 6282     no_overflow(0x7, "vc");
 6283   %}
 6284 %}
 6285 
 6286 // used for unsigned integral comparisons
 6287 
 6288 operand cmpOpU()
 6289 %{
 6290   match(Bool);
 6291 
 6292   format %{ "" %}
 6293   interface(COND_INTER) %{
 6294     equal(0x0, "eq");
 6295     not_equal(0x1, "ne");
 6296     less(0x3, "lo");
 6297     greater_equal(0x2, "hs");
 6298     less_equal(0x9, "ls");
 6299     greater(0x8, "hi");
 6300     overflow(0x6, "vs");
 6301     no_overflow(0x7, "vc");
 6302   %}
 6303 %}
 6304 
 6305 // used for certain integral comparisons which can be
 6306 // converted to cbxx or tbxx instructions
 6307 
 6308 operand cmpOpEqNe()
 6309 %{
 6310   match(Bool);
 6311   op_cost(0);
 6312   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6313             || n->as_Bool()->_test._test == BoolTest::eq);
 6314 
 6315   format %{ "" %}
 6316   interface(COND_INTER) %{
 6317     equal(0x0, "eq");
 6318     not_equal(0x1, "ne");
 6319     less(0xb, "lt");
 6320     greater_equal(0xa, "ge");
 6321     less_equal(0xd, "le");
 6322     greater(0xc, "gt");
 6323     overflow(0x6, "vs");
 6324     no_overflow(0x7, "vc");
 6325   %}
 6326 %}
 6327 
 6328 // used for certain integral comparisons which can be
 6329 // converted to cbxx or tbxx instructions
 6330 
 6331 operand cmpOpLtGe()
 6332 %{
 6333   match(Bool);
 6334   op_cost(0);
 6335 
 6336   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6337             || n->as_Bool()->_test._test == BoolTest::ge);
 6338 
 6339   format %{ "" %}
 6340   interface(COND_INTER) %{
 6341     equal(0x0, "eq");
 6342     not_equal(0x1, "ne");
 6343     less(0xb, "lt");
 6344     greater_equal(0xa, "ge");
 6345     less_equal(0xd, "le");
 6346     greater(0xc, "gt");
 6347     overflow(0x6, "vs");
 6348     no_overflow(0x7, "vc");
 6349   %}
 6350 %}
 6351 
 6352 // used for certain unsigned integral comparisons which can be
 6353 // converted to cbxx or tbxx instructions
 6354 
 6355 operand cmpOpUEqNeLtGe()
 6356 %{
 6357   match(Bool);
 6358   op_cost(0);
 6359 
 6360   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6361             || n->as_Bool()->_test._test == BoolTest::ne
 6362             || n->as_Bool()->_test._test == BoolTest::lt
 6363             || n->as_Bool()->_test._test == BoolTest::ge);
 6364 
 6365   format %{ "" %}
 6366   interface(COND_INTER) %{
 6367     equal(0x0, "eq");
 6368     not_equal(0x1, "ne");
 6369     less(0xb, "lt");
 6370     greater_equal(0xa, "ge");
 6371     less_equal(0xd, "le");
 6372     greater(0xc, "gt");
 6373     overflow(0x6, "vs");
 6374     no_overflow(0x7, "vc");
 6375   %}
 6376 %}
 6377 
 6378 // Special operand allowing long args to int ops to be truncated for free
 6379 
 6380 operand iRegL2I(iRegL reg) %{
 6381 
 6382   op_cost(0);
 6383 
 6384   match(ConvL2I reg);
 6385 
 6386   format %{ "l2i($reg)" %}
 6387 
 6388   interface(REG_INTER)
 6389 %}
 6390 
 6391 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6392 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6393 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6394 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6395 
 6396 //----------OPERAND CLASSES----------------------------------------------------
 6397 // Operand Classes are groups of operands that are used as to simplify
 6398 // instruction definitions by not requiring the AD writer to specify
 6399 // separate instructions for every form of operand when the
 6400 // instruction accepts multiple operand types with the same basic
 6401 // encoding and format. The classic case of this is memory operands.
 6402 
 6403 // memory is used to define read/write location for load/store
 6404 // instruction defs. we can turn a memory op into an Address
 6405 
 6406 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6407                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6408 
 6409 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6410                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6411 
 6412 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6413                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6414 
 6415 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6416                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6417 
 6418 // All of the memory operands. For the pipeline description.
 6419 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6420                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6421                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6422 
 6423 
 6424 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6425 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6426 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6427 // can be elided because the 32-bit instruction will just employ the
 6428 // lower 32 bits anyway.
 6429 //
 6430 // n.b. this does not elide all L2I conversions. if the truncated
 6431 // value is consumed by more than one operation then the ConvL2I
 6432 // cannot be bundled into the consuming nodes so an l2i gets planted
 6433 // (actually a movw $dst $src) and the downstream instructions consume
 6434 // the result of the l2i as an iRegI input. That's a shame since the
 6435 // movw is actually redundant but its not too costly.
 6436 
 6437 opclass iRegIorL2I(iRegI, iRegL2I);
 6438 
 6439 //----------PIPELINE-----------------------------------------------------------
 6440 // Rules which define the behavior of the target architectures pipeline.
 6441 
 6442 // For specific pipelines, eg A53, define the stages of that pipeline
 6443 //pipe_desc(ISS, EX1, EX2, WR);
 6444 #define ISS S0
 6445 #define EX1 S1
 6446 #define EX2 S2
 6447 #define WR  S3
 6448 
 6449 // Integer ALU reg operation
 6450 pipeline %{
 6451 
 6452 attributes %{
 6453   // ARM instructions are of fixed length
 6454   fixed_size_instructions;        // Fixed size instructions TODO does
 6455   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6456   // ARM instructions come in 32-bit word units
 6457   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6458   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6459   instruction_fetch_units = 1;       // of 64 bytes
 6460 
 6461   // List of nop instructions
 6462   nops( MachNop );
 6463 %}
 6464 
 6465 // We don't use an actual pipeline model so don't care about resources
 6466 // or description. we do use pipeline classes to introduce fixed
 6467 // latencies
 6468 
 6469 //----------RESOURCES----------------------------------------------------------
 6470 // Resources are the functional units available to the machine
 6471 
 6472 resources( INS0, INS1, INS01 = INS0 | INS1,
 6473            ALU0, ALU1, ALU = ALU0 | ALU1,
 6474            MAC,
 6475            DIV,
 6476            BRANCH,
 6477            LDST,
 6478            NEON_FP);
 6479 
 6480 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6481 // Pipeline Description specifies the stages in the machine's pipeline
 6482 
 6483 // Define the pipeline as a generic 6 stage pipeline
 6484 pipe_desc(S0, S1, S2, S3, S4, S5);
 6485 
 6486 //----------PIPELINE CLASSES---------------------------------------------------
 6487 // Pipeline Classes describe the stages in which input and output are
 6488 // referenced by the hardware pipeline.
 6489 
 6490 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6491 %{
 6492   single_instruction;
 6493   src1   : S1(read);
 6494   src2   : S2(read);
 6495   dst    : S5(write);
 6496   INS01  : ISS;
 6497   NEON_FP : S5;
 6498 %}
 6499 
 6500 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6501 %{
 6502   single_instruction;
 6503   src1   : S1(read);
 6504   src2   : S2(read);
 6505   dst    : S5(write);
 6506   INS01  : ISS;
 6507   NEON_FP : S5;
 6508 %}
 6509 
 6510 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6511 %{
 6512   single_instruction;
 6513   src    : S1(read);
 6514   dst    : S5(write);
 6515   INS01  : ISS;
 6516   NEON_FP : S5;
 6517 %}
 6518 
 6519 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6520 %{
 6521   single_instruction;
 6522   src    : S1(read);
 6523   dst    : S5(write);
 6524   INS01  : ISS;
 6525   NEON_FP : S5;
 6526 %}
 6527 
 6528 pipe_class fp_d2f(vRegF dst, vRegD src)
 6529 %{
 6530   single_instruction;
 6531   src    : S1(read);
 6532   dst    : S5(write);
 6533   INS01  : ISS;
 6534   NEON_FP : S5;
 6535 %}
 6536 
 6537 pipe_class fp_f2d(vRegD dst, vRegF src)
 6538 %{
 6539   single_instruction;
 6540   src    : S1(read);
 6541   dst    : S5(write);
 6542   INS01  : ISS;
 6543   NEON_FP : S5;
 6544 %}
 6545 
 6546 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6547 %{
 6548   single_instruction;
 6549   src    : S1(read);
 6550   dst    : S5(write);
 6551   INS01  : ISS;
 6552   NEON_FP : S5;
 6553 %}
 6554 
 6555 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6556 %{
 6557   single_instruction;
 6558   src    : S1(read);
 6559   dst    : S5(write);
 6560   INS01  : ISS;
 6561   NEON_FP : S5;
 6562 %}
 6563 
 6564 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6565 %{
 6566   single_instruction;
 6567   src    : S1(read);
 6568   dst    : S5(write);
 6569   INS01  : ISS;
 6570   NEON_FP : S5;
 6571 %}
 6572 
 6573 pipe_class fp_l2f(vRegF dst, iRegL src)
 6574 %{
 6575   single_instruction;
 6576   src    : S1(read);
 6577   dst    : S5(write);
 6578   INS01  : ISS;
 6579   NEON_FP : S5;
 6580 %}
 6581 
 6582 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6583 %{
 6584   single_instruction;
 6585   src    : S1(read);
 6586   dst    : S5(write);
 6587   INS01  : ISS;
 6588   NEON_FP : S5;
 6589 %}
 6590 
 6591 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6592 %{
 6593   single_instruction;
 6594   src    : S1(read);
 6595   dst    : S5(write);
 6596   INS01  : ISS;
 6597   NEON_FP : S5;
 6598 %}
 6599 
 6600 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6601 %{
 6602   single_instruction;
 6603   src    : S1(read);
 6604   dst    : S5(write);
 6605   INS01  : ISS;
 6606   NEON_FP : S5;
 6607 %}
 6608 
 6609 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6610 %{
 6611   single_instruction;
 6612   src    : S1(read);
 6613   dst    : S5(write);
 6614   INS01  : ISS;
 6615   NEON_FP : S5;
 6616 %}
 6617 
 6618 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6619 %{
 6620   single_instruction;
 6621   src1   : S1(read);
 6622   src2   : S2(read);
 6623   dst    : S5(write);
 6624   INS0   : ISS;
 6625   NEON_FP : S5;
 6626 %}
 6627 
 6628 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6629 %{
 6630   single_instruction;
 6631   src1   : S1(read);
 6632   src2   : S2(read);
 6633   dst    : S5(write);
 6634   INS0   : ISS;
 6635   NEON_FP : S5;
 6636 %}
 6637 
 6638 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6639 %{
 6640   single_instruction;
 6641   cr     : S1(read);
 6642   src1   : S1(read);
 6643   src2   : S1(read);
 6644   dst    : S3(write);
 6645   INS01  : ISS;
 6646   NEON_FP : S3;
 6647 %}
 6648 
 6649 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6650 %{
 6651   single_instruction;
 6652   cr     : S1(read);
 6653   src1   : S1(read);
 6654   src2   : S1(read);
 6655   dst    : S3(write);
 6656   INS01  : ISS;
 6657   NEON_FP : S3;
 6658 %}
 6659 
 6660 pipe_class fp_imm_s(vRegF dst)
 6661 %{
 6662   single_instruction;
 6663   dst    : S3(write);
 6664   INS01  : ISS;
 6665   NEON_FP : S3;
 6666 %}
 6667 
 6668 pipe_class fp_imm_d(vRegD dst)
 6669 %{
 6670   single_instruction;
 6671   dst    : S3(write);
 6672   INS01  : ISS;
 6673   NEON_FP : S3;
 6674 %}
 6675 
 6676 pipe_class fp_load_constant_s(vRegF dst)
 6677 %{
 6678   single_instruction;
 6679   dst    : S4(write);
 6680   INS01  : ISS;
 6681   NEON_FP : S4;
 6682 %}
 6683 
 6684 pipe_class fp_load_constant_d(vRegD dst)
 6685 %{
 6686   single_instruction;
 6687   dst    : S4(write);
 6688   INS01  : ISS;
 6689   NEON_FP : S4;
 6690 %}
 6691 
 6692 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6693 %{
 6694   single_instruction;
 6695   dst    : S5(write);
 6696   src1   : S1(read);
 6697   src2   : S1(read);
 6698   INS01  : ISS;
 6699   NEON_FP : S5;
 6700 %}
 6701 
 6702 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6703 %{
 6704   single_instruction;
 6705   dst    : S5(write);
 6706   src1   : S1(read);
 6707   src2   : S1(read);
 6708   INS0   : ISS;
 6709   NEON_FP : S5;
 6710 %}
 6711 
 6712 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6713 %{
 6714   single_instruction;
 6715   dst    : S5(write);
 6716   src1   : S1(read);
 6717   src2   : S1(read);
 6718   dst    : S1(read);
 6719   INS01  : ISS;
 6720   NEON_FP : S5;
 6721 %}
 6722 
 6723 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6724 %{
 6725   single_instruction;
 6726   dst    : S5(write);
 6727   src1   : S1(read);
 6728   src2   : S1(read);
 6729   dst    : S1(read);
 6730   INS0   : ISS;
 6731   NEON_FP : S5;
 6732 %}
 6733 
 6734 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6735 %{
 6736   single_instruction;
 6737   dst    : S4(write);
 6738   src1   : S2(read);
 6739   src2   : S2(read);
 6740   INS01  : ISS;
 6741   NEON_FP : S4;
 6742 %}
 6743 
 6744 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6745 %{
 6746   single_instruction;
 6747   dst    : S4(write);
 6748   src1   : S2(read);
 6749   src2   : S2(read);
 6750   INS0   : ISS;
 6751   NEON_FP : S4;
 6752 %}
 6753 
 6754 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6755 %{
 6756   single_instruction;
 6757   dst    : S3(write);
 6758   src1   : S2(read);
 6759   src2   : S2(read);
 6760   INS01  : ISS;
 6761   NEON_FP : S3;
 6762 %}
 6763 
 6764 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6765 %{
 6766   single_instruction;
 6767   dst    : S3(write);
 6768   src1   : S2(read);
 6769   src2   : S2(read);
 6770   INS0   : ISS;
 6771   NEON_FP : S3;
 6772 %}
 6773 
 6774 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6775 %{
 6776   single_instruction;
 6777   dst    : S3(write);
 6778   src    : S1(read);
 6779   shift  : S1(read);
 6780   INS01  : ISS;
 6781   NEON_FP : S3;
 6782 %}
 6783 
 6784 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6785 %{
 6786   single_instruction;
 6787   dst    : S3(write);
 6788   src    : S1(read);
 6789   shift  : S1(read);
 6790   INS0   : ISS;
 6791   NEON_FP : S3;
 6792 %}
 6793 
 6794 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6795 %{
 6796   single_instruction;
 6797   dst    : S3(write);
 6798   src    : S1(read);
 6799   INS01  : ISS;
 6800   NEON_FP : S3;
 6801 %}
 6802 
 6803 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6804 %{
 6805   single_instruction;
 6806   dst    : S3(write);
 6807   src    : S1(read);
 6808   INS0   : ISS;
 6809   NEON_FP : S3;
 6810 %}
 6811 
 6812 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6813 %{
 6814   single_instruction;
 6815   dst    : S5(write);
 6816   src1   : S1(read);
 6817   src2   : S1(read);
 6818   INS01  : ISS;
 6819   NEON_FP : S5;
 6820 %}
 6821 
 6822 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6823 %{
 6824   single_instruction;
 6825   dst    : S5(write);
 6826   src1   : S1(read);
 6827   src2   : S1(read);
 6828   INS0   : ISS;
 6829   NEON_FP : S5;
 6830 %}
 6831 
 6832 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6833 %{
 6834   single_instruction;
 6835   dst    : S5(write);
 6836   src1   : S1(read);
 6837   src2   : S1(read);
 6838   INS0   : ISS;
 6839   NEON_FP : S5;
 6840 %}
 6841 
 6842 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6843 %{
 6844   single_instruction;
 6845   dst    : S5(write);
 6846   src1   : S1(read);
 6847   src2   : S1(read);
 6848   INS0   : ISS;
 6849   NEON_FP : S5;
 6850 %}
 6851 
 6852 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6853 %{
 6854   single_instruction;
 6855   dst    : S5(write);
 6856   src    : S1(read);
 6857   INS0   : ISS;
 6858   NEON_FP : S5;
 6859 %}
 6860 
 6861 pipe_class vunop_fp64(vecD dst, vecD src)
 6862 %{
 6863   single_instruction;
 6864   dst    : S5(write);
 6865   src    : S1(read);
 6866   INS01  : ISS;
 6867   NEON_FP : S5;
 6868 %}
 6869 
 6870 pipe_class vunop_fp128(vecX dst, vecX src)
 6871 %{
 6872   single_instruction;
 6873   dst    : S5(write);
 6874   src    : S1(read);
 6875   INS0   : ISS;
 6876   NEON_FP : S5;
 6877 %}
 6878 
 6879 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6880 %{
 6881   single_instruction;
 6882   dst    : S3(write);
 6883   src    : S1(read);
 6884   INS01  : ISS;
 6885   NEON_FP : S3;
 6886 %}
 6887 
 6888 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6889 %{
 6890   single_instruction;
 6891   dst    : S3(write);
 6892   src    : S1(read);
 6893   INS01  : ISS;
 6894   NEON_FP : S3;
 6895 %}
 6896 
 6897 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6898 %{
 6899   single_instruction;
 6900   dst    : S3(write);
 6901   src    : S1(read);
 6902   INS01  : ISS;
 6903   NEON_FP : S3;
 6904 %}
 6905 
 6906 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6907 %{
 6908   single_instruction;
 6909   dst    : S3(write);
 6910   src    : S1(read);
 6911   INS01  : ISS;
 6912   NEON_FP : S3;
 6913 %}
 6914 
 6915 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6916 %{
 6917   single_instruction;
 6918   dst    : S3(write);
 6919   src    : S1(read);
 6920   INS01  : ISS;
 6921   NEON_FP : S3;
 6922 %}
 6923 
 6924 pipe_class vmovi_reg_imm64(vecD dst)
 6925 %{
 6926   single_instruction;
 6927   dst    : S3(write);
 6928   INS01  : ISS;
 6929   NEON_FP : S3;
 6930 %}
 6931 
 6932 pipe_class vmovi_reg_imm128(vecX dst)
 6933 %{
 6934   single_instruction;
 6935   dst    : S3(write);
 6936   INS0   : ISS;
 6937   NEON_FP : S3;
 6938 %}
 6939 
 6940 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6941 %{
 6942   single_instruction;
 6943   dst    : S5(write);
 6944   mem    : ISS(read);
 6945   INS01  : ISS;
 6946   NEON_FP : S3;
 6947 %}
 6948 
 6949 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6950 %{
 6951   single_instruction;
 6952   dst    : S5(write);
 6953   mem    : ISS(read);
 6954   INS01  : ISS;
 6955   NEON_FP : S3;
 6956 %}
 6957 
 6958 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6959 %{
 6960   single_instruction;
 6961   mem    : ISS(read);
 6962   src    : S2(read);
 6963   INS01  : ISS;
 6964   NEON_FP : S3;
 6965 %}
 6966 
 6967 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6968 %{
 6969   single_instruction;
 6970   mem    : ISS(read);
 6971   src    : S2(read);
 6972   INS01  : ISS;
 6973   NEON_FP : S3;
 6974 %}
 6975 
 6976 //------- Integer ALU operations --------------------------
 6977 
 6978 // Integer ALU reg-reg operation
 6979 // Operands needed in EX1, result generated in EX2
 6980 // Eg.  ADD     x0, x1, x2
 6981 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6982 %{
 6983   single_instruction;
 6984   dst    : EX2(write);
 6985   src1   : EX1(read);
 6986   src2   : EX1(read);
 6987   INS01  : ISS; // Dual issue as instruction 0 or 1
 6988   ALU    : EX2;
 6989 %}
 6990 
 6991 // Integer ALU reg-reg operation with constant shift
 6992 // Shifted register must be available in LATE_ISS instead of EX1
 6993 // Eg.  ADD     x0, x1, x2, LSL #2
 6994 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6995 %{
 6996   single_instruction;
 6997   dst    : EX2(write);
 6998   src1   : EX1(read);
 6999   src2   : ISS(read);
 7000   INS01  : ISS;
 7001   ALU    : EX2;
 7002 %}
 7003 
 7004 // Integer ALU reg operation with constant shift
 7005 // Eg.  LSL     x0, x1, #shift
 7006 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 7007 %{
 7008   single_instruction;
 7009   dst    : EX2(write);
 7010   src1   : ISS(read);
 7011   INS01  : ISS;
 7012   ALU    : EX2;
 7013 %}
 7014 
 7015 // Integer ALU reg-reg operation with variable shift
 7016 // Both operands must be available in LATE_ISS instead of EX1
 7017 // Result is available in EX1 instead of EX2
 7018 // Eg.  LSLV    x0, x1, x2
 7019 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 7020 %{
 7021   single_instruction;
 7022   dst    : EX1(write);
 7023   src1   : ISS(read);
 7024   src2   : ISS(read);
 7025   INS01  : ISS;
 7026   ALU    : EX1;
 7027 %}
 7028 
 7029 // Integer ALU reg-reg operation with extract
 7030 // As for _vshift above, but result generated in EX2
 7031 // Eg.  EXTR    x0, x1, x2, #N
 7032 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 7033 %{
 7034   single_instruction;
 7035   dst    : EX2(write);
 7036   src1   : ISS(read);
 7037   src2   : ISS(read);
 7038   INS1   : ISS; // Can only dual issue as Instruction 1
 7039   ALU    : EX1;
 7040 %}
 7041 
 7042 // Integer ALU reg operation
 7043 // Eg.  NEG     x0, x1
 7044 pipe_class ialu_reg(iRegI dst, iRegI src)
 7045 %{
 7046   single_instruction;
 7047   dst    : EX2(write);
 7048   src    : EX1(read);
 7049   INS01  : ISS;
 7050   ALU    : EX2;
 7051 %}
 7052 
 7053 // Integer ALU reg mmediate operation
 7054 // Eg.  ADD     x0, x1, #N
 7055 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 7056 %{
 7057   single_instruction;
 7058   dst    : EX2(write);
 7059   src1   : EX1(read);
 7060   INS01  : ISS;
 7061   ALU    : EX2;
 7062 %}
 7063 
 7064 // Integer ALU immediate operation (no source operands)
 7065 // Eg.  MOV     x0, #N
 7066 pipe_class ialu_imm(iRegI dst)
 7067 %{
 7068   single_instruction;
 7069   dst    : EX1(write);
 7070   INS01  : ISS;
 7071   ALU    : EX1;
 7072 %}
 7073 
 7074 //------- Compare operation -------------------------------
 7075 
 7076 // Compare reg-reg
 7077 // Eg.  CMP     x0, x1
 7078 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 7079 %{
 7080   single_instruction;
 7081 //  fixed_latency(16);
 7082   cr     : EX2(write);
 7083   op1    : EX1(read);
 7084   op2    : EX1(read);
 7085   INS01  : ISS;
 7086   ALU    : EX2;
 7087 %}
 7088 
 7089 // Compare reg-reg
 7090 // Eg.  CMP     x0, #N
 7091 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 7092 %{
 7093   single_instruction;
 7094 //  fixed_latency(16);
 7095   cr     : EX2(write);
 7096   op1    : EX1(read);
 7097   INS01  : ISS;
 7098   ALU    : EX2;
 7099 %}
 7100 
 7101 //------- Conditional instructions ------------------------
 7102 
 7103 // Conditional no operands
 7104 // Eg.  CSINC   x0, zr, zr, <cond>
 7105 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 7106 %{
 7107   single_instruction;
 7108   cr     : EX1(read);
 7109   dst    : EX2(write);
 7110   INS01  : ISS;
 7111   ALU    : EX2;
 7112 %}
 7113 
 7114 // Conditional 2 operand
 7115 // EG.  CSEL    X0, X1, X2, <cond>
 7116 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 7117 %{
 7118   single_instruction;
 7119   cr     : EX1(read);
 7120   src1   : EX1(read);
 7121   src2   : EX1(read);
 7122   dst    : EX2(write);
 7123   INS01  : ISS;
 7124   ALU    : EX2;
 7125 %}
 7126 
 7127 // Conditional 2 operand
 7128 // EG.  CSEL    X0, X1, X2, <cond>
 7129 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 7130 %{
 7131   single_instruction;
 7132   cr     : EX1(read);
 7133   src    : EX1(read);
 7134   dst    : EX2(write);
 7135   INS01  : ISS;
 7136   ALU    : EX2;
 7137 %}
 7138 
 7139 //------- Multiply pipeline operations --------------------
 7140 
 7141 // Multiply reg-reg
 7142 // Eg.  MUL     w0, w1, w2
 7143 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7144 %{
 7145   single_instruction;
 7146   dst    : WR(write);
 7147   src1   : ISS(read);
 7148   src2   : ISS(read);
 7149   INS01  : ISS;
 7150   MAC    : WR;
 7151 %}
 7152 
 7153 // Multiply accumulate
 7154 // Eg.  MADD    w0, w1, w2, w3
 7155 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7156 %{
 7157   single_instruction;
 7158   dst    : WR(write);
 7159   src1   : ISS(read);
 7160   src2   : ISS(read);
 7161   src3   : ISS(read);
 7162   INS01  : ISS;
 7163   MAC    : WR;
 7164 %}
 7165 
 7166 // Eg.  MUL     w0, w1, w2
 7167 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7168 %{
 7169   single_instruction;
 7170   fixed_latency(3); // Maximum latency for 64 bit mul
 7171   dst    : WR(write);
 7172   src1   : ISS(read);
 7173   src2   : ISS(read);
 7174   INS01  : ISS;
 7175   MAC    : WR;
 7176 %}
 7177 
 7178 // Multiply accumulate
 7179 // Eg.  MADD    w0, w1, w2, w3
 7180 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7181 %{
 7182   single_instruction;
 7183   fixed_latency(3); // Maximum latency for 64 bit mul
 7184   dst    : WR(write);
 7185   src1   : ISS(read);
 7186   src2   : ISS(read);
 7187   src3   : ISS(read);
 7188   INS01  : ISS;
 7189   MAC    : WR;
 7190 %}
 7191 
 7192 //------- Divide pipeline operations --------------------
 7193 
 7194 // Eg.  SDIV    w0, w1, w2
 7195 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7196 %{
 7197   single_instruction;
 7198   fixed_latency(8); // Maximum latency for 32 bit divide
 7199   dst    : WR(write);
 7200   src1   : ISS(read);
 7201   src2   : ISS(read);
 7202   INS0   : ISS; // Can only dual issue as instruction 0
 7203   DIV    : WR;
 7204 %}
 7205 
 7206 // Eg.  SDIV    x0, x1, x2
 7207 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7208 %{
 7209   single_instruction;
 7210   fixed_latency(16); // Maximum latency for 64 bit divide
 7211   dst    : WR(write);
 7212   src1   : ISS(read);
 7213   src2   : ISS(read);
 7214   INS0   : ISS; // Can only dual issue as instruction 0
 7215   DIV    : WR;
 7216 %}
 7217 
 7218 //------- Load pipeline operations ------------------------
 7219 
 7220 // Load - prefetch
 7221 // Eg.  PFRM    <mem>
 7222 pipe_class iload_prefetch(memory mem)
 7223 %{
 7224   single_instruction;
 7225   mem    : ISS(read);
 7226   INS01  : ISS;
 7227   LDST   : WR;
 7228 %}
 7229 
 7230 // Load - reg, mem
 7231 // Eg.  LDR     x0, <mem>
 7232 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7233 %{
 7234   single_instruction;
 7235   dst    : WR(write);
 7236   mem    : ISS(read);
 7237   INS01  : ISS;
 7238   LDST   : WR;
 7239 %}
 7240 
 7241 // Load - reg, reg
 7242 // Eg.  LDR     x0, [sp, x1]
 7243 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7244 %{
 7245   single_instruction;
 7246   dst    : WR(write);
 7247   src    : ISS(read);
 7248   INS01  : ISS;
 7249   LDST   : WR;
 7250 %}
 7251 
 7252 //------- Store pipeline operations -----------------------
 7253 
 7254 // Store - zr, mem
 7255 // Eg.  STR     zr, <mem>
 7256 pipe_class istore_mem(memory mem)
 7257 %{
 7258   single_instruction;
 7259   mem    : ISS(read);
 7260   INS01  : ISS;
 7261   LDST   : WR;
 7262 %}
 7263 
 7264 // Store - reg, mem
 7265 // Eg.  STR     x0, <mem>
 7266 pipe_class istore_reg_mem(iRegI src, memory mem)
 7267 %{
 7268   single_instruction;
 7269   mem    : ISS(read);
 7270   src    : EX2(read);
 7271   INS01  : ISS;
 7272   LDST   : WR;
 7273 %}
 7274 
 7275 // Store - reg, reg
 7276 // Eg. STR      x0, [sp, x1]
 7277 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7278 %{
 7279   single_instruction;
 7280   dst    : ISS(read);
 7281   src    : EX2(read);
 7282   INS01  : ISS;
 7283   LDST   : WR;
 7284 %}
 7285 
 7286 //------- Store pipeline operations -----------------------
 7287 
 7288 // Branch
 7289 pipe_class pipe_branch()
 7290 %{
 7291   single_instruction;
 7292   INS01  : ISS;
 7293   BRANCH : EX1;
 7294 %}
 7295 
 7296 // Conditional branch
 7297 pipe_class pipe_branch_cond(rFlagsReg cr)
 7298 %{
 7299   single_instruction;
 7300   cr     : EX1(read);
 7301   INS01  : ISS;
 7302   BRANCH : EX1;
 7303 %}
 7304 
 7305 // Compare & Branch
 7306 // EG.  CBZ/CBNZ
 7307 pipe_class pipe_cmp_branch(iRegI op1)
 7308 %{
 7309   single_instruction;
 7310   op1    : EX1(read);
 7311   INS01  : ISS;
 7312   BRANCH : EX1;
 7313 %}
 7314 
 7315 //------- Synchronisation operations ----------------------
 7316 
 7317 // Any operation requiring serialization.
 7318 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7319 pipe_class pipe_serial()
 7320 %{
 7321   single_instruction;
 7322   force_serialization;
 7323   fixed_latency(16);
 7324   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7325   LDST   : WR;
 7326 %}
 7327 
 7328 // Generic big/slow expanded idiom - also serialized
 7329 pipe_class pipe_slow()
 7330 %{
 7331   instruction_count(10);
 7332   multiple_bundles;
 7333   force_serialization;
 7334   fixed_latency(16);
 7335   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7336   LDST   : WR;
 7337 %}
 7338 
 7339 // Empty pipeline class
 7340 pipe_class pipe_class_empty()
 7341 %{
 7342   single_instruction;
 7343   fixed_latency(0);
 7344 %}
 7345 
 7346 // Default pipeline class.
 7347 pipe_class pipe_class_default()
 7348 %{
 7349   single_instruction;
 7350   fixed_latency(2);
 7351 %}
 7352 
 7353 // Pipeline class for compares.
 7354 pipe_class pipe_class_compare()
 7355 %{
 7356   single_instruction;
 7357   fixed_latency(16);
 7358 %}
 7359 
 7360 // Pipeline class for memory operations.
 7361 pipe_class pipe_class_memory()
 7362 %{
 7363   single_instruction;
 7364   fixed_latency(16);
 7365 %}
 7366 
 7367 // Pipeline class for call.
 7368 pipe_class pipe_class_call()
 7369 %{
 7370   single_instruction;
 7371   fixed_latency(100);
 7372 %}
 7373 
 7374 // Define the class for the Nop node.
 7375 define %{
 7376    MachNop = pipe_class_empty;
 7377 %}
 7378 
 7379 %}
 7380 //----------INSTRUCTIONS-------------------------------------------------------
 7381 //
 7382 // match      -- States which machine-independent subtree may be replaced
 7383 //               by this instruction.
 7384 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7385 //               selection to identify a minimum cost tree of machine
 7386 //               instructions that matches a tree of machine-independent
 7387 //               instructions.
 7388 // format     -- A string providing the disassembly for this instruction.
 7389 //               The value of an instruction's operand may be inserted
 7390 //               by referring to it with a '$' prefix.
 7391 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7392 //               to within an encode class as $primary, $secondary, and $tertiary
 7393 //               rrspectively.  The primary opcode is commonly used to
 7394 //               indicate the type of machine instruction, while secondary
 7395 //               and tertiary are often used for prefix options or addressing
 7396 //               modes.
 7397 // ins_encode -- A list of encode classes with parameters. The encode class
 7398 //               name must have been defined in an 'enc_class' specification
 7399 //               in the encode section of the architecture description.
 7400 
 7401 // ============================================================================
 7402 // Memory (Load/Store) Instructions
 7403 
 7404 // Load Instructions
 7405 
 7406 // Load Byte (8 bit signed)
 7407 instruct loadB(iRegINoSp dst, memory1 mem)
 7408 %{
 7409   match(Set dst (LoadB mem));
 7410   predicate(!needs_acquiring_load(n));
 7411 
 7412   ins_cost(4 * INSN_COST);
 7413   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7414 
 7415   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7416 
 7417   ins_pipe(iload_reg_mem);
 7418 %}
 7419 
 7420 // Load Byte (8 bit signed) into long
 7421 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7422 %{
 7423   match(Set dst (ConvI2L (LoadB mem)));
 7424   predicate(!needs_acquiring_load(n->in(1)));
 7425 
 7426   ins_cost(4 * INSN_COST);
 7427   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7428 
 7429   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7430 
 7431   ins_pipe(iload_reg_mem);
 7432 %}
 7433 
 7434 // Load Byte (8 bit unsigned)
 7435 instruct loadUB(iRegINoSp dst, memory1 mem)
 7436 %{
 7437   match(Set dst (LoadUB mem));
 7438   predicate(!needs_acquiring_load(n));
 7439 
 7440   ins_cost(4 * INSN_COST);
 7441   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7442 
 7443   ins_encode(aarch64_enc_ldrb(dst, mem));
 7444 
 7445   ins_pipe(iload_reg_mem);
 7446 %}
 7447 
 7448 // Load Byte (8 bit unsigned) into long
 7449 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7450 %{
 7451   match(Set dst (ConvI2L (LoadUB mem)));
 7452   predicate(!needs_acquiring_load(n->in(1)));
 7453 
 7454   ins_cost(4 * INSN_COST);
 7455   format %{ "ldrb  $dst, $mem\t# byte" %}
 7456 
 7457   ins_encode(aarch64_enc_ldrb(dst, mem));
 7458 
 7459   ins_pipe(iload_reg_mem);
 7460 %}
 7461 
 7462 // Load Short (16 bit signed)
 7463 instruct loadS(iRegINoSp dst, memory2 mem)
 7464 %{
 7465   match(Set dst (LoadS mem));
 7466   predicate(!needs_acquiring_load(n));
 7467 
 7468   ins_cost(4 * INSN_COST);
 7469   format %{ "ldrshw  $dst, $mem\t# short" %}
 7470 
 7471   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7472 
 7473   ins_pipe(iload_reg_mem);
 7474 %}
 7475 
 7476 // Load Short (16 bit signed) into long
 7477 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7478 %{
 7479   match(Set dst (ConvI2L (LoadS mem)));
 7480   predicate(!needs_acquiring_load(n->in(1)));
 7481 
 7482   ins_cost(4 * INSN_COST);
 7483   format %{ "ldrsh  $dst, $mem\t# short" %}
 7484 
 7485   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7486 
 7487   ins_pipe(iload_reg_mem);
 7488 %}
 7489 
 7490 // Load Char (16 bit unsigned)
 7491 instruct loadUS(iRegINoSp dst, memory2 mem)
 7492 %{
 7493   match(Set dst (LoadUS mem));
 7494   predicate(!needs_acquiring_load(n));
 7495 
 7496   ins_cost(4 * INSN_COST);
 7497   format %{ "ldrh  $dst, $mem\t# short" %}
 7498 
 7499   ins_encode(aarch64_enc_ldrh(dst, mem));
 7500 
 7501   ins_pipe(iload_reg_mem);
 7502 %}
 7503 
 7504 // Load Short/Char (16 bit unsigned) into long
 7505 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7506 %{
 7507   match(Set dst (ConvI2L (LoadUS mem)));
 7508   predicate(!needs_acquiring_load(n->in(1)));
 7509 
 7510   ins_cost(4 * INSN_COST);
 7511   format %{ "ldrh  $dst, $mem\t# short" %}
 7512 
 7513   ins_encode(aarch64_enc_ldrh(dst, mem));
 7514 
 7515   ins_pipe(iload_reg_mem);
 7516 %}
 7517 
 7518 // Load Integer (32 bit signed)
 7519 instruct loadI(iRegINoSp dst, memory4 mem)
 7520 %{
 7521   match(Set dst (LoadI mem));
 7522   predicate(!needs_acquiring_load(n));
 7523 
 7524   ins_cost(4 * INSN_COST);
 7525   format %{ "ldrw  $dst, $mem\t# int" %}
 7526 
 7527   ins_encode(aarch64_enc_ldrw(dst, mem));
 7528 
 7529   ins_pipe(iload_reg_mem);
 7530 %}
 7531 
 7532 // Load Integer (32 bit signed) into long
 7533 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7534 %{
 7535   match(Set dst (ConvI2L (LoadI mem)));
 7536   predicate(!needs_acquiring_load(n->in(1)));
 7537 
 7538   ins_cost(4 * INSN_COST);
 7539   format %{ "ldrsw  $dst, $mem\t# int" %}
 7540 
 7541   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7542 
 7543   ins_pipe(iload_reg_mem);
 7544 %}
 7545 
 7546 // Load Integer (32 bit unsigned) into long
 7547 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7548 %{
 7549   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7550   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7551 
 7552   ins_cost(4 * INSN_COST);
 7553   format %{ "ldrw  $dst, $mem\t# int" %}
 7554 
 7555   ins_encode(aarch64_enc_ldrw(dst, mem));
 7556 
 7557   ins_pipe(iload_reg_mem);
 7558 %}
 7559 
 7560 // Load Long (64 bit signed)
 7561 instruct loadL(iRegLNoSp dst, memory8 mem)
 7562 %{
 7563   match(Set dst (LoadL mem));
 7564   predicate(!needs_acquiring_load(n));
 7565 
 7566   ins_cost(4 * INSN_COST);
 7567   format %{ "ldr  $dst, $mem\t# int" %}
 7568 
 7569   ins_encode(aarch64_enc_ldr(dst, mem));
 7570 
 7571   ins_pipe(iload_reg_mem);
 7572 %}
 7573 
 7574 // Load Range
 7575 instruct loadRange(iRegINoSp dst, memory4 mem)
 7576 %{
 7577   match(Set dst (LoadRange mem));
 7578 
 7579   ins_cost(4 * INSN_COST);
 7580   format %{ "ldrw  $dst, $mem\t# range" %}
 7581 
 7582   ins_encode(aarch64_enc_ldrw(dst, mem));
 7583 
 7584   ins_pipe(iload_reg_mem);
 7585 %}
 7586 
 7587 // Load Pointer
 7588 instruct loadP(iRegPNoSp dst, memory8 mem)
 7589 %{
 7590   match(Set dst (LoadP mem));
 7591   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7592 
 7593   ins_cost(4 * INSN_COST);
 7594   format %{ "ldr  $dst, $mem\t# ptr" %}
 7595 
 7596   ins_encode(aarch64_enc_ldr(dst, mem));
 7597 
 7598   ins_pipe(iload_reg_mem);
 7599 %}
 7600 
 7601 // Load Compressed Pointer
 7602 instruct loadN(iRegNNoSp dst, memory4 mem)
 7603 %{
 7604   match(Set dst (LoadN mem));
 7605   predicate(!needs_acquiring_load(n));
 7606 
 7607   ins_cost(4 * INSN_COST);
 7608   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7609 
 7610   ins_encode(aarch64_enc_ldrw(dst, mem));
 7611 
 7612   ins_pipe(iload_reg_mem);
 7613 %}
 7614 
 7615 // Load Klass Pointer
 7616 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7617 %{
 7618   match(Set dst (LoadKlass mem));
 7619   predicate(!needs_acquiring_load(n));
 7620 
 7621   ins_cost(4 * INSN_COST);
 7622   format %{ "ldr  $dst, $mem\t# class" %}
 7623 
 7624   ins_encode(aarch64_enc_ldr(dst, mem));
 7625 
 7626   ins_pipe(iload_reg_mem);
 7627 %}
 7628 
 7629 // Load Narrow Klass Pointer
 7630 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7631 %{
 7632   match(Set dst (LoadNKlass mem));
 7633   predicate(!needs_acquiring_load(n));
 7634 
 7635   ins_cost(4 * INSN_COST);
 7636   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7637 
 7638   ins_encode(aarch64_enc_ldrw(dst, mem));
 7639 
 7640   ins_pipe(iload_reg_mem);
 7641 %}
 7642 
 7643 // Load Float
 7644 instruct loadF(vRegF dst, memory4 mem)
 7645 %{
 7646   match(Set dst (LoadF mem));
 7647   predicate(!needs_acquiring_load(n));
 7648 
 7649   ins_cost(4 * INSN_COST);
 7650   format %{ "ldrs  $dst, $mem\t# float" %}
 7651 
 7652   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7653 
 7654   ins_pipe(pipe_class_memory);
 7655 %}
 7656 
 7657 // Load Double
 7658 instruct loadD(vRegD dst, memory8 mem)
 7659 %{
 7660   match(Set dst (LoadD mem));
 7661   predicate(!needs_acquiring_load(n));
 7662 
 7663   ins_cost(4 * INSN_COST);
 7664   format %{ "ldrd  $dst, $mem\t# double" %}
 7665 
 7666   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7667 
 7668   ins_pipe(pipe_class_memory);
 7669 %}
 7670 
 7671 
 7672 // Load Int Constant
 7673 instruct loadConI(iRegINoSp dst, immI src)
 7674 %{
 7675   match(Set dst src);
 7676 
 7677   ins_cost(INSN_COST);
 7678   format %{ "mov $dst, $src\t# int" %}
 7679 
 7680   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7681 
 7682   ins_pipe(ialu_imm);
 7683 %}
 7684 
 7685 // Load Long Constant
 7686 instruct loadConL(iRegLNoSp dst, immL src)
 7687 %{
 7688   match(Set dst src);
 7689 
 7690   ins_cost(INSN_COST);
 7691   format %{ "mov $dst, $src\t# long" %}
 7692 
 7693   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7694 
 7695   ins_pipe(ialu_imm);
 7696 %}
 7697 
 7698 // Load Pointer Constant
 7699 
 7700 instruct loadConP(iRegPNoSp dst, immP con)
 7701 %{
 7702   match(Set dst con);
 7703 
 7704   ins_cost(INSN_COST * 4);
 7705   format %{
 7706     "mov  $dst, $con\t# ptr\n\t"
 7707   %}
 7708 
 7709   ins_encode(aarch64_enc_mov_p(dst, con));
 7710 
 7711   ins_pipe(ialu_imm);
 7712 %}
 7713 
 7714 // Load Null Pointer Constant
 7715 
 7716 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7717 %{
 7718   match(Set dst con);
 7719 
 7720   ins_cost(INSN_COST);
 7721   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7722 
 7723   ins_encode(aarch64_enc_mov_p0(dst, con));
 7724 
 7725   ins_pipe(ialu_imm);
 7726 %}
 7727 
 7728 // Load Pointer Constant One
 7729 
 7730 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7731 %{
 7732   match(Set dst con);
 7733 
 7734   ins_cost(INSN_COST);
 7735   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7736 
 7737   ins_encode(aarch64_enc_mov_p1(dst, con));
 7738 
 7739   ins_pipe(ialu_imm);
 7740 %}
 7741 
 7742 // Load Byte Map Base Constant
 7743 
 7744 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7745 %{
 7746   match(Set dst con);
 7747 
 7748   ins_cost(INSN_COST);
 7749   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7750 
 7751   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7752 
 7753   ins_pipe(ialu_imm);
 7754 %}
 7755 
 7756 // Load Narrow Pointer Constant
 7757 
 7758 instruct loadConN(iRegNNoSp dst, immN con)
 7759 %{
 7760   match(Set dst con);
 7761 
 7762   ins_cost(INSN_COST * 4);
 7763   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7764 
 7765   ins_encode(aarch64_enc_mov_n(dst, con));
 7766 
 7767   ins_pipe(ialu_imm);
 7768 %}
 7769 
 7770 // Load Narrow Null Pointer Constant
 7771 
 7772 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7773 %{
 7774   match(Set dst con);
 7775 
 7776   ins_cost(INSN_COST);
 7777   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7778 
 7779   ins_encode(aarch64_enc_mov_n0(dst, con));
 7780 
 7781   ins_pipe(ialu_imm);
 7782 %}
 7783 
 7784 // Load Narrow Klass Constant
 7785 
 7786 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7787 %{
 7788   match(Set dst con);
 7789 
 7790   ins_cost(INSN_COST);
 7791   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7792 
 7793   ins_encode(aarch64_enc_mov_nk(dst, con));
 7794 
 7795   ins_pipe(ialu_imm);
 7796 %}
 7797 
 7798 // Load Packed Float Constant
 7799 
 7800 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7801   match(Set dst con);
 7802   ins_cost(INSN_COST * 4);
 7803   format %{ "fmovs  $dst, $con"%}
 7804   ins_encode %{
 7805     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7806   %}
 7807 
 7808   ins_pipe(fp_imm_s);
 7809 %}
 7810 
 7811 // Load Float Constant
 7812 
 7813 instruct loadConF(vRegF dst, immF con) %{
 7814   match(Set dst con);
 7815 
 7816   ins_cost(INSN_COST * 4);
 7817 
 7818   format %{
 7819     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7820   %}
 7821 
 7822   ins_encode %{
 7823     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7824   %}
 7825 
 7826   ins_pipe(fp_load_constant_s);
 7827 %}
 7828 
 7829 // Load Packed Double Constant
 7830 
 7831 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7832   match(Set dst con);
 7833   ins_cost(INSN_COST);
 7834   format %{ "fmovd  $dst, $con"%}
 7835   ins_encode %{
 7836     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7837   %}
 7838 
 7839   ins_pipe(fp_imm_d);
 7840 %}
 7841 
 7842 // Load Double Constant
 7843 
 7844 instruct loadConD(vRegD dst, immD con) %{
 7845   match(Set dst con);
 7846 
 7847   ins_cost(INSN_COST * 5);
 7848   format %{
 7849     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7850   %}
 7851 
 7852   ins_encode %{
 7853     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7854   %}
 7855 
 7856   ins_pipe(fp_load_constant_d);
 7857 %}
 7858 
 7859 // Store Instructions
 7860 
 7861 // Store CMS card-mark Immediate
 7862 instruct storeimmCM0(immI0 zero, memory1 mem)
 7863 %{
 7864   match(Set mem (StoreCM mem zero));
 7865 
 7866   ins_cost(INSN_COST);
 7867   format %{ "storestore (elided)\n\t"
 7868             "strb zr, $mem\t# byte" %}
 7869 
 7870   ins_encode(aarch64_enc_strb0(mem));
 7871 
 7872   ins_pipe(istore_mem);
 7873 %}
 7874 
 7875 // Store CMS card-mark Immediate with intervening StoreStore
 7876 // needed when using CMS with no conditional card marking
 7877 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7878 %{
 7879   match(Set mem (StoreCM mem zero));
 7880 
 7881   ins_cost(INSN_COST * 2);
 7882   format %{ "storestore\n\t"
 7883             "dmb ishst"
 7884             "\n\tstrb zr, $mem\t# byte" %}
 7885 
 7886   ins_encode(aarch64_enc_strb0_ordered(mem));
 7887 
 7888   ins_pipe(istore_mem);
 7889 %}
 7890 
 7891 // Store Byte
 7892 instruct storeB(iRegIorL2I src, memory1 mem)
 7893 %{
 7894   match(Set mem (StoreB mem src));
 7895   predicate(!needs_releasing_store(n));
 7896 
 7897   ins_cost(INSN_COST);
 7898   format %{ "strb  $src, $mem\t# byte" %}
 7899 
 7900   ins_encode(aarch64_enc_strb(src, mem));
 7901 
 7902   ins_pipe(istore_reg_mem);
 7903 %}
 7904 
 7905 
 7906 instruct storeimmB0(immI0 zero, memory1 mem)
 7907 %{
 7908   match(Set mem (StoreB mem zero));
 7909   predicate(!needs_releasing_store(n));
 7910 
 7911   ins_cost(INSN_COST);
 7912   format %{ "strb rscractch2, $mem\t# byte" %}
 7913 
 7914   ins_encode(aarch64_enc_strb0(mem));
 7915 
 7916   ins_pipe(istore_mem);
 7917 %}
 7918 
 7919 // Store Char/Short
 7920 instruct storeC(iRegIorL2I src, memory2 mem)
 7921 %{
 7922   match(Set mem (StoreC mem src));
 7923   predicate(!needs_releasing_store(n));
 7924 
 7925   ins_cost(INSN_COST);
 7926   format %{ "strh  $src, $mem\t# short" %}
 7927 
 7928   ins_encode(aarch64_enc_strh(src, mem));
 7929 
 7930   ins_pipe(istore_reg_mem);
 7931 %}
 7932 
 7933 instruct storeimmC0(immI0 zero, memory2 mem)
 7934 %{
 7935   match(Set mem (StoreC mem zero));
 7936   predicate(!needs_releasing_store(n));
 7937 
 7938   ins_cost(INSN_COST);
 7939   format %{ "strh  zr, $mem\t# short" %}
 7940 
 7941   ins_encode(aarch64_enc_strh0(mem));
 7942 
 7943   ins_pipe(istore_mem);
 7944 %}
 7945 
 7946 // Store Integer
 7947 
 7948 instruct storeI(iRegIorL2I src, memory4 mem)
 7949 %{
 7950   match(Set mem(StoreI mem src));
 7951   predicate(!needs_releasing_store(n));
 7952 
 7953   ins_cost(INSN_COST);
 7954   format %{ "strw  $src, $mem\t# int" %}
 7955 
 7956   ins_encode(aarch64_enc_strw(src, mem));
 7957 
 7958   ins_pipe(istore_reg_mem);
 7959 %}
 7960 
 7961 instruct storeimmI0(immI0 zero, memory4 mem)
 7962 %{
 7963   match(Set mem(StoreI mem zero));
 7964   predicate(!needs_releasing_store(n));
 7965 
 7966   ins_cost(INSN_COST);
 7967   format %{ "strw  zr, $mem\t# int" %}
 7968 
 7969   ins_encode(aarch64_enc_strw0(mem));
 7970 
 7971   ins_pipe(istore_mem);
 7972 %}
 7973 
 7974 // Store Long (64 bit signed)
 7975 instruct storeL(iRegL src, memory8 mem)
 7976 %{
 7977   match(Set mem (StoreL mem src));
 7978   predicate(!needs_releasing_store(n));
 7979 
 7980   ins_cost(INSN_COST);
 7981   format %{ "str  $src, $mem\t# int" %}
 7982 
 7983   ins_encode(aarch64_enc_str(src, mem));
 7984 
 7985   ins_pipe(istore_reg_mem);
 7986 %}
 7987 
 7988 // Store Long (64 bit signed)
 7989 instruct storeimmL0(immL0 zero, memory8 mem)
 7990 %{
 7991   match(Set mem (StoreL mem zero));
 7992   predicate(!needs_releasing_store(n));
 7993 
 7994   ins_cost(INSN_COST);
 7995   format %{ "str  zr, $mem\t# int" %}
 7996 
 7997   ins_encode(aarch64_enc_str0(mem));
 7998 
 7999   ins_pipe(istore_mem);
 8000 %}
 8001 
 8002 // Store Pointer
 8003 instruct storeP(iRegP src, memory8 mem)
 8004 %{
 8005   match(Set mem (StoreP mem src));
 8006   predicate(!needs_releasing_store(n));
 8007 
 8008   ins_cost(INSN_COST);
 8009   format %{ "str  $src, $mem\t# ptr" %}
 8010 
 8011   ins_encode(aarch64_enc_str(src, mem));
 8012 
 8013   ins_pipe(istore_reg_mem);
 8014 %}
 8015 
 8016 // Store Pointer
 8017 instruct storeimmP0(immP0 zero, memory8 mem)
 8018 %{
 8019   match(Set mem (StoreP mem zero));
 8020   predicate(!needs_releasing_store(n));
 8021 
 8022   ins_cost(INSN_COST);
 8023   format %{ "str zr, $mem\t# ptr" %}
 8024 
 8025   ins_encode(aarch64_enc_str0(mem));
 8026 
 8027   ins_pipe(istore_mem);
 8028 %}
 8029 
 8030 // Store Compressed Pointer
 8031 instruct storeN(iRegN src, memory4 mem)
 8032 %{
 8033   match(Set mem (StoreN mem src));
 8034   predicate(!needs_releasing_store(n));
 8035 
 8036   ins_cost(INSN_COST);
 8037   format %{ "strw  $src, $mem\t# compressed ptr" %}
 8038 
 8039   ins_encode(aarch64_enc_strw(src, mem));
 8040 
 8041   ins_pipe(istore_reg_mem);
 8042 %}
 8043 
 8044 instruct storeImmN0(immN0 zero, memory4 mem)
 8045 %{
 8046   match(Set mem (StoreN mem zero));
 8047   predicate(!needs_releasing_store(n));
 8048 
 8049   ins_cost(INSN_COST);
 8050   format %{ "strw  zr, $mem\t# compressed ptr" %}
 8051 
 8052   ins_encode(aarch64_enc_strw0(mem));
 8053 
 8054   ins_pipe(istore_mem);
 8055 %}
 8056 
 8057 // Store Float
 8058 instruct storeF(vRegF src, memory4 mem)
 8059 %{
 8060   match(Set mem (StoreF mem src));
 8061   predicate(!needs_releasing_store(n));
 8062 
 8063   ins_cost(INSN_COST);
 8064   format %{ "strs  $src, $mem\t# float" %}
 8065 
 8066   ins_encode( aarch64_enc_strs(src, mem) );
 8067 
 8068   ins_pipe(pipe_class_memory);
 8069 %}
 8070 
 8071 // TODO
 8072 // implement storeImmF0 and storeFImmPacked
 8073 
 8074 // Store Double
 8075 instruct storeD(vRegD src, memory8 mem)
 8076 %{
 8077   match(Set mem (StoreD mem src));
 8078   predicate(!needs_releasing_store(n));
 8079 
 8080   ins_cost(INSN_COST);
 8081   format %{ "strd  $src, $mem\t# double" %}
 8082 
 8083   ins_encode( aarch64_enc_strd(src, mem) );
 8084 
 8085   ins_pipe(pipe_class_memory);
 8086 %}
 8087 
 8088 // Store Compressed Klass Pointer
 8089 instruct storeNKlass(iRegN src, memory4 mem)
 8090 %{
 8091   predicate(!needs_releasing_store(n));
 8092   match(Set mem (StoreNKlass mem src));
 8093 
 8094   ins_cost(INSN_COST);
 8095   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 8096 
 8097   ins_encode(aarch64_enc_strw(src, mem));
 8098 
 8099   ins_pipe(istore_reg_mem);
 8100 %}
 8101 
 8102 // TODO
 8103 // implement storeImmD0 and storeDImmPacked
 8104 
 8105 // prefetch instructions
 8106 // Must be safe to execute with invalid address (cannot fault).
 8107 
 8108 instruct prefetchalloc( memory8 mem ) %{
 8109   match(PrefetchAllocation mem);
 8110 
 8111   ins_cost(INSN_COST);
 8112   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 8113 
 8114   ins_encode( aarch64_enc_prefetchw(mem) );
 8115 
 8116   ins_pipe(iload_prefetch);
 8117 %}
 8118 
 8119 //  ---------------- volatile loads and stores ----------------
 8120 
 8121 // Load Byte (8 bit signed)
 8122 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8123 %{
 8124   match(Set dst (LoadB mem));
 8125 
 8126   ins_cost(VOLATILE_REF_COST);
 8127   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8128 
 8129   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8130 
 8131   ins_pipe(pipe_serial);
 8132 %}
 8133 
 8134 // Load Byte (8 bit signed) into long
 8135 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8136 %{
 8137   match(Set dst (ConvI2L (LoadB mem)));
 8138 
 8139   ins_cost(VOLATILE_REF_COST);
 8140   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8141 
 8142   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8143 
 8144   ins_pipe(pipe_serial);
 8145 %}
 8146 
 8147 // Load Byte (8 bit unsigned)
 8148 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8149 %{
 8150   match(Set dst (LoadUB mem));
 8151 
 8152   ins_cost(VOLATILE_REF_COST);
 8153   format %{ "ldarb  $dst, $mem\t# byte" %}
 8154 
 8155   ins_encode(aarch64_enc_ldarb(dst, mem));
 8156 
 8157   ins_pipe(pipe_serial);
 8158 %}
 8159 
 8160 // Load Byte (8 bit unsigned) into long
 8161 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8162 %{
 8163   match(Set dst (ConvI2L (LoadUB mem)));
 8164 
 8165   ins_cost(VOLATILE_REF_COST);
 8166   format %{ "ldarb  $dst, $mem\t# byte" %}
 8167 
 8168   ins_encode(aarch64_enc_ldarb(dst, mem));
 8169 
 8170   ins_pipe(pipe_serial);
 8171 %}
 8172 
 8173 // Load Short (16 bit signed)
 8174 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8175 %{
 8176   match(Set dst (LoadS mem));
 8177 
 8178   ins_cost(VOLATILE_REF_COST);
 8179   format %{ "ldarshw  $dst, $mem\t# short" %}
 8180 
 8181   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8182 
 8183   ins_pipe(pipe_serial);
 8184 %}
 8185 
 8186 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8187 %{
 8188   match(Set dst (LoadUS mem));
 8189 
 8190   ins_cost(VOLATILE_REF_COST);
 8191   format %{ "ldarhw  $dst, $mem\t# short" %}
 8192 
 8193   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8194 
 8195   ins_pipe(pipe_serial);
 8196 %}
 8197 
 8198 // Load Short/Char (16 bit unsigned) into long
 8199 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8200 %{
 8201   match(Set dst (ConvI2L (LoadUS mem)));
 8202 
 8203   ins_cost(VOLATILE_REF_COST);
 8204   format %{ "ldarh  $dst, $mem\t# short" %}
 8205 
 8206   ins_encode(aarch64_enc_ldarh(dst, mem));
 8207 
 8208   ins_pipe(pipe_serial);
 8209 %}
 8210 
 8211 // Load Short/Char (16 bit signed) into long
 8212 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8213 %{
 8214   match(Set dst (ConvI2L (LoadS mem)));
 8215 
 8216   ins_cost(VOLATILE_REF_COST);
 8217   format %{ "ldarh  $dst, $mem\t# short" %}
 8218 
 8219   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8220 
 8221   ins_pipe(pipe_serial);
 8222 %}
 8223 
 8224 // Load Integer (32 bit signed)
 8225 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8226 %{
 8227   match(Set dst (LoadI mem));
 8228 
 8229   ins_cost(VOLATILE_REF_COST);
 8230   format %{ "ldarw  $dst, $mem\t# int" %}
 8231 
 8232   ins_encode(aarch64_enc_ldarw(dst, mem));
 8233 
 8234   ins_pipe(pipe_serial);
 8235 %}
 8236 
 8237 // Load Integer (32 bit unsigned) into long
 8238 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8239 %{
 8240   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8241 
 8242   ins_cost(VOLATILE_REF_COST);
 8243   format %{ "ldarw  $dst, $mem\t# int" %}
 8244 
 8245   ins_encode(aarch64_enc_ldarw(dst, mem));
 8246 
 8247   ins_pipe(pipe_serial);
 8248 %}
 8249 
 8250 // Load Long (64 bit signed)
 8251 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8252 %{
 8253   match(Set dst (LoadL mem));
 8254 
 8255   ins_cost(VOLATILE_REF_COST);
 8256   format %{ "ldar  $dst, $mem\t# int" %}
 8257 
 8258   ins_encode(aarch64_enc_ldar(dst, mem));
 8259 
 8260   ins_pipe(pipe_serial);
 8261 %}
 8262 
 8263 // Load Pointer
 8264 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8265 %{
 8266   match(Set dst (LoadP mem));
 8267   predicate(n->as_Load()->barrier_data() == 0);
 8268 
 8269   ins_cost(VOLATILE_REF_COST);
 8270   format %{ "ldar  $dst, $mem\t# ptr" %}
 8271 
 8272   ins_encode(aarch64_enc_ldar(dst, mem));
 8273 
 8274   ins_pipe(pipe_serial);
 8275 %}
 8276 
 8277 // Load Compressed Pointer
 8278 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8279 %{
 8280   match(Set dst (LoadN mem));
 8281 
 8282   ins_cost(VOLATILE_REF_COST);
 8283   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8284 
 8285   ins_encode(aarch64_enc_ldarw(dst, mem));
 8286 
 8287   ins_pipe(pipe_serial);
 8288 %}
 8289 
 8290 // Load Float
 8291 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8292 %{
 8293   match(Set dst (LoadF mem));
 8294 
 8295   ins_cost(VOLATILE_REF_COST);
 8296   format %{ "ldars  $dst, $mem\t# float" %}
 8297 
 8298   ins_encode( aarch64_enc_fldars(dst, mem) );
 8299 
 8300   ins_pipe(pipe_serial);
 8301 %}
 8302 
 8303 // Load Double
 8304 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8305 %{
 8306   match(Set dst (LoadD mem));
 8307 
 8308   ins_cost(VOLATILE_REF_COST);
 8309   format %{ "ldard  $dst, $mem\t# double" %}
 8310 
 8311   ins_encode( aarch64_enc_fldard(dst, mem) );
 8312 
 8313   ins_pipe(pipe_serial);
 8314 %}
 8315 
 8316 // Store Byte
 8317 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8318 %{
 8319   match(Set mem (StoreB mem src));
 8320 
 8321   ins_cost(VOLATILE_REF_COST);
 8322   format %{ "stlrb  $src, $mem\t# byte" %}
 8323 
 8324   ins_encode(aarch64_enc_stlrb(src, mem));
 8325 
 8326   ins_pipe(pipe_class_memory);
 8327 %}
 8328 
 8329 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8330 %{
 8331   match(Set mem (StoreB mem zero));
 8332 
 8333   ins_cost(VOLATILE_REF_COST);
 8334   format %{ "stlrb  zr, $mem\t# byte" %}
 8335 
 8336   ins_encode(aarch64_enc_stlrb0(mem));
 8337 
 8338   ins_pipe(pipe_class_memory);
 8339 %}
 8340 
 8341 // Store Char/Short
 8342 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8343 %{
 8344   match(Set mem (StoreC mem src));
 8345 
 8346   ins_cost(VOLATILE_REF_COST);
 8347   format %{ "stlrh  $src, $mem\t# short" %}
 8348 
 8349   ins_encode(aarch64_enc_stlrh(src, mem));
 8350 
 8351   ins_pipe(pipe_class_memory);
 8352 %}
 8353 
 8354 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8355 %{
 8356   match(Set mem (StoreC mem zero));
 8357 
 8358   ins_cost(VOLATILE_REF_COST);
 8359   format %{ "stlrh  zr, $mem\t# short" %}
 8360 
 8361   ins_encode(aarch64_enc_stlrh0(mem));
 8362 
 8363   ins_pipe(pipe_class_memory);
 8364 %}
 8365 
 8366 // Store Integer
 8367 
 8368 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8369 %{
 8370   match(Set mem(StoreI mem src));
 8371 
 8372   ins_cost(VOLATILE_REF_COST);
 8373   format %{ "stlrw  $src, $mem\t# int" %}
 8374 
 8375   ins_encode(aarch64_enc_stlrw(src, mem));
 8376 
 8377   ins_pipe(pipe_class_memory);
 8378 %}
 8379 
 8380 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8381 %{
 8382   match(Set mem(StoreI mem zero));
 8383 
 8384   ins_cost(VOLATILE_REF_COST);
 8385   format %{ "stlrw  zr, $mem\t# int" %}
 8386 
 8387   ins_encode(aarch64_enc_stlrw0(mem));
 8388 
 8389   ins_pipe(pipe_class_memory);
 8390 %}
 8391 
 8392 // Store Long (64 bit signed)
 8393 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8394 %{
 8395   match(Set mem (StoreL mem src));
 8396 
 8397   ins_cost(VOLATILE_REF_COST);
 8398   format %{ "stlr  $src, $mem\t# int" %}
 8399 
 8400   ins_encode(aarch64_enc_stlr(src, mem));
 8401 
 8402   ins_pipe(pipe_class_memory);
 8403 %}
 8404 
 8405 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8406 %{
 8407   match(Set mem (StoreL mem zero));
 8408 
 8409   ins_cost(VOLATILE_REF_COST);
 8410   format %{ "stlr  zr, $mem\t# int" %}
 8411 
 8412   ins_encode(aarch64_enc_stlr0(mem));
 8413 
 8414   ins_pipe(pipe_class_memory);
 8415 %}
 8416 
 8417 // Store Pointer
 8418 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8419 %{
 8420   match(Set mem (StoreP mem src));
 8421 
 8422   ins_cost(VOLATILE_REF_COST);
 8423   format %{ "stlr  $src, $mem\t# ptr" %}
 8424 
 8425   ins_encode(aarch64_enc_stlr(src, mem));
 8426 
 8427   ins_pipe(pipe_class_memory);
 8428 %}
 8429 
 8430 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8431 %{
 8432   match(Set mem (StoreP mem zero));
 8433 
 8434   ins_cost(VOLATILE_REF_COST);
 8435   format %{ "stlr  zr, $mem\t# ptr" %}
 8436 
 8437   ins_encode(aarch64_enc_stlr0(mem));
 8438 
 8439   ins_pipe(pipe_class_memory);
 8440 %}
 8441 
 8442 // Store Compressed Pointer
 8443 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8444 %{
 8445   match(Set mem (StoreN mem src));
 8446 
 8447   ins_cost(VOLATILE_REF_COST);
 8448   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8449 
 8450   ins_encode(aarch64_enc_stlrw(src, mem));
 8451 
 8452   ins_pipe(pipe_class_memory);
 8453 %}
 8454 
 8455 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8456 %{
 8457   match(Set mem (StoreN mem zero));
 8458 
 8459   ins_cost(VOLATILE_REF_COST);
 8460   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8461 
 8462   ins_encode(aarch64_enc_stlrw0(mem));
 8463 
 8464   ins_pipe(pipe_class_memory);
 8465 %}
 8466 
 8467 // Store Float
 8468 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8469 %{
 8470   match(Set mem (StoreF mem src));
 8471 
 8472   ins_cost(VOLATILE_REF_COST);
 8473   format %{ "stlrs  $src, $mem\t# float" %}
 8474 
 8475   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8476 
 8477   ins_pipe(pipe_class_memory);
 8478 %}
 8479 
 8480 // TODO
 8481 // implement storeImmF0 and storeFImmPacked
 8482 
 8483 // Store Double
 8484 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8485 %{
 8486   match(Set mem (StoreD mem src));
 8487 
 8488   ins_cost(VOLATILE_REF_COST);
 8489   format %{ "stlrd  $src, $mem\t# double" %}
 8490 
 8491   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8492 
 8493   ins_pipe(pipe_class_memory);
 8494 %}
 8495 
 8496 //  ---------------- end of volatile loads and stores ----------------
 8497 
 8498 instruct cacheWB(indirect addr)
 8499 %{
 8500   predicate(VM_Version::supports_data_cache_line_flush());
 8501   match(CacheWB addr);
 8502 
 8503   ins_cost(100);
 8504   format %{"cache wb $addr" %}
 8505   ins_encode %{
 8506     assert($addr->index_position() < 0, "should be");
 8507     assert($addr$$disp == 0, "should be");
 8508     __ cache_wb(Address($addr$$base$$Register, 0));
 8509   %}
 8510   ins_pipe(pipe_slow); // XXX
 8511 %}
 8512 
 8513 instruct cacheWBPreSync()
 8514 %{
 8515   predicate(VM_Version::supports_data_cache_line_flush());
 8516   match(CacheWBPreSync);
 8517 
 8518   ins_cost(100);
 8519   format %{"cache wb presync" %}
 8520   ins_encode %{
 8521     __ cache_wbsync(true);
 8522   %}
 8523   ins_pipe(pipe_slow); // XXX
 8524 %}
 8525 
 8526 instruct cacheWBPostSync()
 8527 %{
 8528   predicate(VM_Version::supports_data_cache_line_flush());
 8529   match(CacheWBPostSync);
 8530 
 8531   ins_cost(100);
 8532   format %{"cache wb postsync" %}
 8533   ins_encode %{
 8534     __ cache_wbsync(false);
 8535   %}
 8536   ins_pipe(pipe_slow); // XXX
 8537 %}
 8538 
 8539 // ============================================================================
 8540 // BSWAP Instructions
 8541 
 8542 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8543   match(Set dst (ReverseBytesI src));
 8544 
 8545   ins_cost(INSN_COST);
 8546   format %{ "revw  $dst, $src" %}
 8547 
 8548   ins_encode %{
 8549     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8550   %}
 8551 
 8552   ins_pipe(ialu_reg);
 8553 %}
 8554 
 8555 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8556   match(Set dst (ReverseBytesL src));
 8557 
 8558   ins_cost(INSN_COST);
 8559   format %{ "rev  $dst, $src" %}
 8560 
 8561   ins_encode %{
 8562     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8563   %}
 8564 
 8565   ins_pipe(ialu_reg);
 8566 %}
 8567 
 8568 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8569   match(Set dst (ReverseBytesUS src));
 8570 
 8571   ins_cost(INSN_COST);
 8572   format %{ "rev16w  $dst, $src" %}
 8573 
 8574   ins_encode %{
 8575     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8576   %}
 8577 
 8578   ins_pipe(ialu_reg);
 8579 %}
 8580 
 8581 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8582   match(Set dst (ReverseBytesS src));
 8583 
 8584   ins_cost(INSN_COST);
 8585   format %{ "rev16w  $dst, $src\n\t"
 8586             "sbfmw $dst, $dst, #0, #15" %}
 8587 
 8588   ins_encode %{
 8589     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8590     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8591   %}
 8592 
 8593   ins_pipe(ialu_reg);
 8594 %}
 8595 
 8596 // ============================================================================
 8597 // Zero Count Instructions
 8598 
 8599 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8600   match(Set dst (CountLeadingZerosI src));
 8601 
 8602   ins_cost(INSN_COST);
 8603   format %{ "clzw  $dst, $src" %}
 8604   ins_encode %{
 8605     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8606   %}
 8607 
 8608   ins_pipe(ialu_reg);
 8609 %}
 8610 
 8611 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8612   match(Set dst (CountLeadingZerosL src));
 8613 
 8614   ins_cost(INSN_COST);
 8615   format %{ "clz   $dst, $src" %}
 8616   ins_encode %{
 8617     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8618   %}
 8619 
 8620   ins_pipe(ialu_reg);
 8621 %}
 8622 
 8623 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8624   match(Set dst (CountTrailingZerosI src));
 8625 
 8626   ins_cost(INSN_COST * 2);
 8627   format %{ "rbitw  $dst, $src\n\t"
 8628             "clzw   $dst, $dst" %}
 8629   ins_encode %{
 8630     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8631     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8632   %}
 8633 
 8634   ins_pipe(ialu_reg);
 8635 %}
 8636 
 8637 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8638   match(Set dst (CountTrailingZerosL src));
 8639 
 8640   ins_cost(INSN_COST * 2);
 8641   format %{ "rbit   $dst, $src\n\t"
 8642             "clz    $dst, $dst" %}
 8643   ins_encode %{
 8644     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8645     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8646   %}
 8647 
 8648   ins_pipe(ialu_reg);
 8649 %}
 8650 
 8651 //---------- Population Count Instructions -------------------------------------
 8652 //
 8653 
 8654 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8655   match(Set dst (PopCountI src));
 8656   effect(TEMP tmp);
 8657   ins_cost(INSN_COST * 13);
 8658 
 8659   format %{ "movw   $src, $src\n\t"
 8660             "mov    $tmp, $src\t# vector (1D)\n\t"
 8661             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8662             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8663             "mov    $dst, $tmp\t# vector (1D)" %}
 8664   ins_encode %{
 8665     assert(UsePopCountInstruction, "unsupported");
 8666     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8667     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8668     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8669     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8670     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8671   %}
 8672 
 8673   ins_pipe(pipe_class_default);
 8674 %}
 8675 
 8676 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8677   match(Set dst (PopCountI (LoadI mem)));
 8678   effect(TEMP tmp);
 8679   ins_cost(INSN_COST * 13);
 8680 
 8681   format %{ "ldrs   $tmp, $mem\n\t"
 8682             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8683             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8684             "mov    $dst, $tmp\t# vector (1D)" %}
 8685   ins_encode %{
 8686     assert(UsePopCountInstruction, "unsupported");
 8687     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8688     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8689               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8690     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8691     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8692     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8693   %}
 8694 
 8695   ins_pipe(pipe_class_default);
 8696 %}
 8697 
 8698 // Note: Long.bitCount(long) returns an int.
 8699 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8700   match(Set dst (PopCountL src));
 8701   effect(TEMP tmp);
 8702   ins_cost(INSN_COST * 13);
 8703 
 8704   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8705             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8706             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8707             "mov    $dst, $tmp\t# vector (1D)" %}
 8708   ins_encode %{
 8709     assert(UsePopCountInstruction, "unsupported");
 8710     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8711     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8712     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8713     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8714   %}
 8715 
 8716   ins_pipe(pipe_class_default);
 8717 %}
 8718 
 8719 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8720   match(Set dst (PopCountL (LoadL mem)));
 8721   effect(TEMP tmp);
 8722   ins_cost(INSN_COST * 13);
 8723 
 8724   format %{ "ldrd   $tmp, $mem\n\t"
 8725             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8726             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8727             "mov    $dst, $tmp\t# vector (1D)" %}
 8728   ins_encode %{
 8729     assert(UsePopCountInstruction, "unsupported");
 8730     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8731     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8732               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8733     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8734     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8735     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8736   %}
 8737 
 8738   ins_pipe(pipe_class_default);
 8739 %}
 8740 
 8741 // ============================================================================
 8742 // MemBar Instruction
 8743 
 8744 instruct load_fence() %{
 8745   match(LoadFence);
 8746   ins_cost(VOLATILE_REF_COST);
 8747 
 8748   format %{ "load_fence" %}
 8749 
 8750   ins_encode %{
 8751     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8752   %}
 8753   ins_pipe(pipe_serial);
 8754 %}
 8755 
 8756 instruct unnecessary_membar_acquire() %{
 8757   predicate(unnecessary_acquire(n));
 8758   match(MemBarAcquire);
 8759   ins_cost(0);
 8760 
 8761   format %{ "membar_acquire (elided)" %}
 8762 
 8763   ins_encode %{
 8764     __ block_comment("membar_acquire (elided)");
 8765   %}
 8766 
 8767   ins_pipe(pipe_class_empty);
 8768 %}
 8769 
 8770 instruct membar_acquire() %{
 8771   match(MemBarAcquire);
 8772   ins_cost(VOLATILE_REF_COST);
 8773 
 8774   format %{ "membar_acquire\n\t"
 8775             "dmb ish" %}
 8776 
 8777   ins_encode %{
 8778     __ block_comment("membar_acquire");
 8779     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8780   %}
 8781 
 8782   ins_pipe(pipe_serial);
 8783 %}
 8784 
 8785 
 8786 instruct membar_acquire_lock() %{
 8787   match(MemBarAcquireLock);
 8788   ins_cost(VOLATILE_REF_COST);
 8789 
 8790   format %{ "membar_acquire_lock (elided)" %}
 8791 
 8792   ins_encode %{
 8793     __ block_comment("membar_acquire_lock (elided)");
 8794   %}
 8795 
 8796   ins_pipe(pipe_serial);
 8797 %}
 8798 
 8799 instruct store_fence() %{
 8800   match(StoreFence);
 8801   ins_cost(VOLATILE_REF_COST);
 8802 
 8803   format %{ "store_fence" %}
 8804 
 8805   ins_encode %{
 8806     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8807   %}
 8808   ins_pipe(pipe_serial);
 8809 %}
 8810 
 8811 instruct unnecessary_membar_release() %{
 8812   predicate(unnecessary_release(n));
 8813   match(MemBarRelease);
 8814   ins_cost(0);
 8815 
 8816   format %{ "membar_release (elided)" %}
 8817 
 8818   ins_encode %{
 8819     __ block_comment("membar_release (elided)");
 8820   %}
 8821   ins_pipe(pipe_serial);
 8822 %}
 8823 
 8824 instruct membar_release() %{
 8825   match(MemBarRelease);
 8826   ins_cost(VOLATILE_REF_COST);
 8827 
 8828   format %{ "membar_release\n\t"
 8829             "dmb ish" %}
 8830 
 8831   ins_encode %{
 8832     __ block_comment("membar_release");
 8833     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8834   %}
 8835   ins_pipe(pipe_serial);
 8836 %}
 8837 
 8838 instruct membar_storestore() %{
 8839   match(MemBarStoreStore);
 8840   match(StoreStoreFence);
 8841   ins_cost(VOLATILE_REF_COST);
 8842 
 8843   format %{ "MEMBAR-store-store" %}
 8844 
 8845   ins_encode %{
 8846     __ membar(Assembler::StoreStore);
 8847   %}
 8848   ins_pipe(pipe_serial);
 8849 %}
 8850 
 8851 instruct membar_release_lock() %{
 8852   match(MemBarReleaseLock);
 8853   ins_cost(VOLATILE_REF_COST);
 8854 
 8855   format %{ "membar_release_lock (elided)" %}
 8856 
 8857   ins_encode %{
 8858     __ block_comment("membar_release_lock (elided)");
 8859   %}
 8860 
 8861   ins_pipe(pipe_serial);
 8862 %}
 8863 
 8864 instruct unnecessary_membar_volatile() %{
 8865   predicate(unnecessary_volatile(n));
 8866   match(MemBarVolatile);
 8867   ins_cost(0);
 8868 
 8869   format %{ "membar_volatile (elided)" %}
 8870 
 8871   ins_encode %{
 8872     __ block_comment("membar_volatile (elided)");
 8873   %}
 8874 
 8875   ins_pipe(pipe_serial);
 8876 %}
 8877 
 8878 instruct membar_volatile() %{
 8879   match(MemBarVolatile);
 8880   ins_cost(VOLATILE_REF_COST*100);
 8881 
 8882   format %{ "membar_volatile\n\t"
 8883              "dmb ish"%}
 8884 
 8885   ins_encode %{
 8886     __ block_comment("membar_volatile");
 8887     __ membar(Assembler::StoreLoad);
 8888   %}
 8889 
 8890   ins_pipe(pipe_serial);
 8891 %}
 8892 
 8893 // ============================================================================
 8894 // Cast/Convert Instructions
 8895 
 8896 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8897   match(Set dst (CastX2P src));
 8898 
 8899   ins_cost(INSN_COST);
 8900   format %{ "mov $dst, $src\t# long -> ptr" %}
 8901 
 8902   ins_encode %{
 8903     if ($dst$$reg != $src$$reg) {
 8904       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8905     }
 8906   %}
 8907 
 8908   ins_pipe(ialu_reg);
 8909 %}
 8910 
 8911 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8912   match(Set dst (CastP2X src));
 8913 
 8914   ins_cost(INSN_COST);
 8915   format %{ "mov $dst, $src\t# ptr -> long" %}
 8916 
 8917   ins_encode %{
 8918     if ($dst$$reg != $src$$reg) {
 8919       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8920     }
 8921   %}
 8922 
 8923   ins_pipe(ialu_reg);
 8924 %}
 8925 
 8926 // Convert oop into int for vectors alignment masking
 8927 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8928   match(Set dst (ConvL2I (CastP2X src)));
 8929 
 8930   ins_cost(INSN_COST);
 8931   format %{ "movw $dst, $src\t# ptr -> int" %}
 8932   ins_encode %{
 8933     __ movw($dst$$Register, $src$$Register);
 8934   %}
 8935 
 8936   ins_pipe(ialu_reg);
 8937 %}
 8938 
 8939 // Convert compressed oop into int for vectors alignment masking
 8940 // in case of 32bit oops (heap < 4Gb).
 8941 instruct convN2I(iRegINoSp dst, iRegN src)
 8942 %{
 8943   predicate(CompressedOops::shift() == 0);
 8944   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8945 
 8946   ins_cost(INSN_COST);
 8947   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8948   ins_encode %{
 8949     __ movw($dst$$Register, $src$$Register);
 8950   %}
 8951 
 8952   ins_pipe(ialu_reg);
 8953 %}
 8954 
 8955 
 8956 // Convert oop pointer into compressed form
 8957 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8958   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8959   match(Set dst (EncodeP src));
 8960   effect(KILL cr);
 8961   ins_cost(INSN_COST * 3);
 8962   format %{ "encode_heap_oop $dst, $src" %}
 8963   ins_encode %{
 8964     Register s = $src$$Register;
 8965     Register d = $dst$$Register;
 8966     __ encode_heap_oop(d, s);
 8967   %}
 8968   ins_pipe(ialu_reg);
 8969 %}
 8970 
 8971 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8972   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8973   match(Set dst (EncodeP src));
 8974   ins_cost(INSN_COST * 3);
 8975   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8976   ins_encode %{
 8977     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8978   %}
 8979   ins_pipe(ialu_reg);
 8980 %}
 8981 
 8982 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8983   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8984             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8985   match(Set dst (DecodeN src));
 8986   ins_cost(INSN_COST * 3);
 8987   format %{ "decode_heap_oop $dst, $src" %}
 8988   ins_encode %{
 8989     Register s = $src$$Register;
 8990     Register d = $dst$$Register;
 8991     __ decode_heap_oop(d, s);
 8992   %}
 8993   ins_pipe(ialu_reg);
 8994 %}
 8995 
 8996 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8997   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8998             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8999   match(Set dst (DecodeN src));
 9000   ins_cost(INSN_COST * 3);
 9001   format %{ "decode_heap_oop_not_null $dst, $src" %}
 9002   ins_encode %{
 9003     Register s = $src$$Register;
 9004     Register d = $dst$$Register;
 9005     __ decode_heap_oop_not_null(d, s);
 9006   %}
 9007   ins_pipe(ialu_reg);
 9008 %}
 9009 
 9010 // n.b. AArch64 implementations of encode_klass_not_null and
 9011 // decode_klass_not_null do not modify the flags register so, unlike
 9012 // Intel, we don't kill CR as a side effect here
 9013 
 9014 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 9015   match(Set dst (EncodePKlass src));
 9016 
 9017   ins_cost(INSN_COST * 3);
 9018   format %{ "encode_klass_not_null $dst,$src" %}
 9019 
 9020   ins_encode %{
 9021     Register src_reg = as_Register($src$$reg);
 9022     Register dst_reg = as_Register($dst$$reg);
 9023     __ encode_klass_not_null(dst_reg, src_reg);
 9024   %}
 9025 
 9026    ins_pipe(ialu_reg);
 9027 %}
 9028 
 9029 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 9030   match(Set dst (DecodeNKlass src));
 9031 
 9032   ins_cost(INSN_COST * 3);
 9033   format %{ "decode_klass_not_null $dst,$src" %}
 9034 
 9035   ins_encode %{
 9036     Register src_reg = as_Register($src$$reg);
 9037     Register dst_reg = as_Register($dst$$reg);
 9038     if (dst_reg != src_reg) {
 9039       __ decode_klass_not_null(dst_reg, src_reg);
 9040     } else {
 9041       __ decode_klass_not_null(dst_reg);
 9042     }
 9043   %}
 9044 
 9045    ins_pipe(ialu_reg);
 9046 %}
 9047 
 9048 instruct checkCastPP(iRegPNoSp dst)
 9049 %{
 9050   match(Set dst (CheckCastPP dst));
 9051 
 9052   size(0);
 9053   format %{ "# checkcastPP of $dst" %}
 9054   ins_encode(/* empty encoding */);
 9055   ins_pipe(pipe_class_empty);
 9056 %}
 9057 
 9058 instruct castPP(iRegPNoSp dst)
 9059 %{
 9060   match(Set dst (CastPP dst));
 9061 
 9062   size(0);
 9063   format %{ "# castPP of $dst" %}
 9064   ins_encode(/* empty encoding */);
 9065   ins_pipe(pipe_class_empty);
 9066 %}
 9067 
 9068 instruct castII(iRegI dst)
 9069 %{
 9070   match(Set dst (CastII dst));
 9071 
 9072   size(0);
 9073   format %{ "# castII of $dst" %}
 9074   ins_encode(/* empty encoding */);
 9075   ins_cost(0);
 9076   ins_pipe(pipe_class_empty);
 9077 %}
 9078 
 9079 instruct castLL(iRegL dst)
 9080 %{
 9081   match(Set dst (CastLL dst));
 9082 
 9083   size(0);
 9084   format %{ "# castLL of $dst" %}
 9085   ins_encode(/* empty encoding */);
 9086   ins_cost(0);
 9087   ins_pipe(pipe_class_empty);
 9088 %}
 9089 
 9090 instruct castFF(vRegF dst)
 9091 %{
 9092   match(Set dst (CastFF dst));
 9093 
 9094   size(0);
 9095   format %{ "# castFF of $dst" %}
 9096   ins_encode(/* empty encoding */);
 9097   ins_cost(0);
 9098   ins_pipe(pipe_class_empty);
 9099 %}
 9100 
 9101 instruct castDD(vRegD dst)
 9102 %{
 9103   match(Set dst (CastDD dst));
 9104 
 9105   size(0);
 9106   format %{ "# castDD of $dst" %}
 9107   ins_encode(/* empty encoding */);
 9108   ins_cost(0);
 9109   ins_pipe(pipe_class_empty);
 9110 %}
 9111 
 9112 instruct castVVD(vecD dst)
 9113 %{
 9114   match(Set dst (CastVV dst));
 9115 
 9116   size(0);
 9117   format %{ "# castVV of $dst" %}
 9118   ins_encode(/* empty encoding */);
 9119   ins_cost(0);
 9120   ins_pipe(pipe_class_empty);
 9121 %}
 9122 
 9123 instruct castVVX(vecX dst)
 9124 %{
 9125   match(Set dst (CastVV dst));
 9126 
 9127   size(0);
 9128   format %{ "# castVV of $dst" %}
 9129   ins_encode(/* empty encoding */);
 9130   ins_cost(0);
 9131   ins_pipe(pipe_class_empty);
 9132 %}
 9133 
 9134 instruct castVV(vReg dst)
 9135 %{
 9136   match(Set dst (CastVV dst));
 9137 
 9138   size(0);
 9139   format %{ "# castVV of $dst" %}
 9140   ins_encode(/* empty encoding */);
 9141   ins_cost(0);
 9142   ins_pipe(pipe_class_empty);
 9143 %}
 9144 
 9145 instruct castVVMask(pRegGov dst)
 9146 %{
 9147   match(Set dst (CastVV dst));
 9148 
 9149   size(0);
 9150   format %{ "# castVV of $dst" %}
 9151   ins_encode(/* empty encoding */);
 9152   ins_cost(0);
 9153   ins_pipe(pipe_class_empty);
 9154 %}
 9155 
 9156 // ============================================================================
 9157 // Atomic operation instructions
 9158 //
 9159 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9160 // Store{PIL}Conditional instructions using a normal load for the
 9161 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9162 //
 9163 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9164 // pair to lock object allocations from Eden space when not using
 9165 // TLABs.
 9166 //
 9167 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9168 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9169 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9170 // only for 64-bit.
 9171 //
 9172 // We implement LoadPLocked and StorePLocked instructions using,
 9173 // respectively the AArch64 hw load-exclusive and store-conditional
 9174 // instructions. Whereas we must implement each of
 9175 // Store{IL}Conditional using a CAS which employs a pair of
 9176 // instructions comprising a load-exclusive followed by a
 9177 // store-conditional.
 9178 
 9179 
 9180 // Locked-load (linked load) of the current heap-top
 9181 // used when updating the eden heap top
 9182 // implemented using ldaxr on AArch64
 9183 
 9184 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9185 %{
 9186   match(Set dst (LoadPLocked mem));
 9187 
 9188   ins_cost(VOLATILE_REF_COST);
 9189 
 9190   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9191 
 9192   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9193 
 9194   ins_pipe(pipe_serial);
 9195 %}
 9196 
 9197 // Conditional-store of the updated heap-top.
 9198 // Used during allocation of the shared heap.
 9199 // Sets flag (EQ) on success.
 9200 // implemented using stlxr on AArch64.
 9201 
 9202 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9203 %{
 9204   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9205 
 9206   ins_cost(VOLATILE_REF_COST);
 9207 
 9208  // TODO
 9209  // do we need to do a store-conditional release or can we just use a
 9210  // plain store-conditional?
 9211 
 9212   format %{
 9213     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9214     "cmpw rscratch1, zr\t# EQ on successful write"
 9215   %}
 9216 
 9217   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9218 
 9219   ins_pipe(pipe_serial);
 9220 %}
 9221 
 9222 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9223 %{
 9224   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9225 
 9226   ins_cost(VOLATILE_REF_COST);
 9227 
 9228   format %{
 9229     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9230     "cmpw rscratch1, zr\t# EQ on successful write"
 9231   %}
 9232 
 9233   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9234 
 9235   ins_pipe(pipe_slow);
 9236 %}
 9237 
 9238 // storeIConditional also has acquire semantics, for no better reason
 9239 // than matching storeLConditional.  At the time of writing this
 9240 // comment storeIConditional was not used anywhere by AArch64.
 9241 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9242 %{
 9243   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9244 
 9245   ins_cost(VOLATILE_REF_COST);
 9246 
 9247   format %{
 9248     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9249     "cmpw rscratch1, zr\t# EQ on successful write"
 9250   %}
 9251 
 9252   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9253 
 9254   ins_pipe(pipe_slow);
 9255 %}
 9256 
 9257 // standard CompareAndSwapX when we are using barriers
 9258 // these have higher priority than the rules selected by a predicate
 9259 
 9260 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9261 // can't match them
 9262 
 9263 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9264 
 9265   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9266   ins_cost(2 * VOLATILE_REF_COST);
 9267 
 9268   effect(KILL cr);
 9269 
 9270   format %{
 9271     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9272     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9273   %}
 9274 
 9275   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9276             aarch64_enc_cset_eq(res));
 9277 
 9278   ins_pipe(pipe_slow);
 9279 %}
 9280 
 9281 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9282 
 9283   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9284   ins_cost(2 * VOLATILE_REF_COST);
 9285 
 9286   effect(KILL cr);
 9287 
 9288   format %{
 9289     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9290     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9291   %}
 9292 
 9293   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9294             aarch64_enc_cset_eq(res));
 9295 
 9296   ins_pipe(pipe_slow);
 9297 %}
 9298 
 9299 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9300 
 9301   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9302   ins_cost(2 * VOLATILE_REF_COST);
 9303 
 9304   effect(KILL cr);
 9305 
 9306  format %{
 9307     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9308     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9309  %}
 9310 
 9311  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9312             aarch64_enc_cset_eq(res));
 9313 
 9314   ins_pipe(pipe_slow);
 9315 %}
 9316 
 9317 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9318 
 9319   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9320   ins_cost(2 * VOLATILE_REF_COST);
 9321 
 9322   effect(KILL cr);
 9323 
 9324  format %{
 9325     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9326     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9327  %}
 9328 
 9329  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9330             aarch64_enc_cset_eq(res));
 9331 
 9332   ins_pipe(pipe_slow);
 9333 %}
 9334 
 9335 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9336 
 9337   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9338   predicate(n->as_LoadStore()->barrier_data() == 0);
 9339   ins_cost(2 * VOLATILE_REF_COST);
 9340 
 9341   effect(KILL cr);
 9342 
 9343  format %{
 9344     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9345     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9346  %}
 9347 
 9348  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9349             aarch64_enc_cset_eq(res));
 9350 
 9351   ins_pipe(pipe_slow);
 9352 %}
 9353 
 9354 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9355 
 9356   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9357   ins_cost(2 * VOLATILE_REF_COST);
 9358 
 9359   effect(KILL cr);
 9360 
 9361  format %{
 9362     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9363     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9364  %}
 9365 
 9366  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9367             aarch64_enc_cset_eq(res));
 9368 
 9369   ins_pipe(pipe_slow);
 9370 %}
 9371 
 9372 // alternative CompareAndSwapX when we are eliding barriers
 9373 
 9374 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9375 
 9376   predicate(needs_acquiring_load_exclusive(n));
 9377   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9378   ins_cost(VOLATILE_REF_COST);
 9379 
 9380   effect(KILL cr);
 9381 
 9382   format %{
 9383     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9384     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9385   %}
 9386 
 9387   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9388             aarch64_enc_cset_eq(res));
 9389 
 9390   ins_pipe(pipe_slow);
 9391 %}
 9392 
 9393 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9394 
 9395   predicate(needs_acquiring_load_exclusive(n));
 9396   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9397   ins_cost(VOLATILE_REF_COST);
 9398 
 9399   effect(KILL cr);
 9400 
 9401   format %{
 9402     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9403     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9404   %}
 9405 
 9406   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9407             aarch64_enc_cset_eq(res));
 9408 
 9409   ins_pipe(pipe_slow);
 9410 %}
 9411 
 9412 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9413 
 9414   predicate(needs_acquiring_load_exclusive(n));
 9415   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9416   ins_cost(VOLATILE_REF_COST);
 9417 
 9418   effect(KILL cr);
 9419 
 9420  format %{
 9421     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9422     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9423  %}
 9424 
 9425  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9426             aarch64_enc_cset_eq(res));
 9427 
 9428   ins_pipe(pipe_slow);
 9429 %}
 9430 
 9431 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9432 
 9433   predicate(needs_acquiring_load_exclusive(n));
 9434   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9435   ins_cost(VOLATILE_REF_COST);
 9436 
 9437   effect(KILL cr);
 9438 
 9439  format %{
 9440     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9441     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9442  %}
 9443 
 9444  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9445             aarch64_enc_cset_eq(res));
 9446 
 9447   ins_pipe(pipe_slow);
 9448 %}
 9449 
 9450 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9451 
 9452   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9453   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9454   ins_cost(VOLATILE_REF_COST);
 9455 
 9456   effect(KILL cr);
 9457 
 9458  format %{
 9459     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9460     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9461  %}
 9462 
 9463  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9464             aarch64_enc_cset_eq(res));
 9465 
 9466   ins_pipe(pipe_slow);
 9467 %}
 9468 
 9469 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9470 
 9471   predicate(needs_acquiring_load_exclusive(n));
 9472   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9473   ins_cost(VOLATILE_REF_COST);
 9474 
 9475   effect(KILL cr);
 9476 
 9477  format %{
 9478     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9479     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9480  %}
 9481 
 9482  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9483             aarch64_enc_cset_eq(res));
 9484 
 9485   ins_pipe(pipe_slow);
 9486 %}
 9487 
 9488 
 9489 // ---------------------------------------------------------------------
 9490 
 9491 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9492 
 9493 // Sundry CAS operations.  Note that release is always true,
 9494 // regardless of the memory ordering of the CAS.  This is because we
 9495 // need the volatile case to be sequentially consistent but there is
 9496 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9497 // can't check the type of memory ordering here, so we always emit a
 9498 // STLXR.
 9499 
 9500 // This section is generated from aarch64_ad_cas.m4
 9501 
 9502 
 9503 
 9504 // This pattern is generated automatically from cas.m4.
 9505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9506 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9507 
 9508   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9509   ins_cost(2 * VOLATILE_REF_COST);
 9510   effect(TEMP_DEF res, KILL cr);
 9511   format %{
 9512     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9513   %}
 9514   ins_encode %{
 9515     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9516                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9517                /*weak*/ false, $res$$Register);
 9518     __ sxtbw($res$$Register, $res$$Register);
 9519   %}
 9520   ins_pipe(pipe_slow);
 9521 %}
 9522 
 9523 // This pattern is generated automatically from cas.m4.
 9524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9525 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9526 
 9527   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9528   ins_cost(2 * VOLATILE_REF_COST);
 9529   effect(TEMP_DEF res, KILL cr);
 9530   format %{
 9531     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9532   %}
 9533   ins_encode %{
 9534     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9535                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9536                /*weak*/ false, $res$$Register);
 9537     __ sxthw($res$$Register, $res$$Register);
 9538   %}
 9539   ins_pipe(pipe_slow);
 9540 %}
 9541 
 9542 // This pattern is generated automatically from cas.m4.
 9543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9544 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9545 
 9546   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9547   ins_cost(2 * VOLATILE_REF_COST);
 9548   effect(TEMP_DEF res, KILL cr);
 9549   format %{
 9550     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9551   %}
 9552   ins_encode %{
 9553     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9554                Assembler::word, /*acquire*/ false, /*release*/ true,
 9555                /*weak*/ false, $res$$Register);
 9556   %}
 9557   ins_pipe(pipe_slow);
 9558 %}
 9559 
 9560 // This pattern is generated automatically from cas.m4.
 9561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9562 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9563 
 9564   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9565   ins_cost(2 * VOLATILE_REF_COST);
 9566   effect(TEMP_DEF res, KILL cr);
 9567   format %{
 9568     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9569   %}
 9570   ins_encode %{
 9571     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9572                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9573                /*weak*/ false, $res$$Register);
 9574   %}
 9575   ins_pipe(pipe_slow);
 9576 %}
 9577 
 9578 // This pattern is generated automatically from cas.m4.
 9579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9580 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9581 
 9582   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9583   ins_cost(2 * VOLATILE_REF_COST);
 9584   effect(TEMP_DEF res, KILL cr);
 9585   format %{
 9586     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9587   %}
 9588   ins_encode %{
 9589     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9590                Assembler::word, /*acquire*/ false, /*release*/ true,
 9591                /*weak*/ false, $res$$Register);
 9592   %}
 9593   ins_pipe(pipe_slow);
 9594 %}
 9595 
 9596 // This pattern is generated automatically from cas.m4.
 9597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9598 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9599   predicate(n->as_LoadStore()->barrier_data() == 0);
 9600   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9601   ins_cost(2 * VOLATILE_REF_COST);
 9602   effect(TEMP_DEF res, KILL cr);
 9603   format %{
 9604     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9605   %}
 9606   ins_encode %{
 9607     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9608                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9609                /*weak*/ false, $res$$Register);
 9610   %}
 9611   ins_pipe(pipe_slow);
 9612 %}
 9613 
 9614 // This pattern is generated automatically from cas.m4.
 9615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9616 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9617   predicate(needs_acquiring_load_exclusive(n));
 9618   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9619   ins_cost(VOLATILE_REF_COST);
 9620   effect(TEMP_DEF res, KILL cr);
 9621   format %{
 9622     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9623   %}
 9624   ins_encode %{
 9625     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9626                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9627                /*weak*/ false, $res$$Register);
 9628     __ sxtbw($res$$Register, $res$$Register);
 9629   %}
 9630   ins_pipe(pipe_slow);
 9631 %}
 9632 
 9633 // This pattern is generated automatically from cas.m4.
 9634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9635 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9636   predicate(needs_acquiring_load_exclusive(n));
 9637   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9638   ins_cost(VOLATILE_REF_COST);
 9639   effect(TEMP_DEF res, KILL cr);
 9640   format %{
 9641     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9642   %}
 9643   ins_encode %{
 9644     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9645                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9646                /*weak*/ false, $res$$Register);
 9647     __ sxthw($res$$Register, $res$$Register);
 9648   %}
 9649   ins_pipe(pipe_slow);
 9650 %}
 9651 
 9652 // This pattern is generated automatically from cas.m4.
 9653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9654 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9655   predicate(needs_acquiring_load_exclusive(n));
 9656   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9657   ins_cost(VOLATILE_REF_COST);
 9658   effect(TEMP_DEF res, KILL cr);
 9659   format %{
 9660     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9661   %}
 9662   ins_encode %{
 9663     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9664                Assembler::word, /*acquire*/ true, /*release*/ true,
 9665                /*weak*/ false, $res$$Register);
 9666   %}
 9667   ins_pipe(pipe_slow);
 9668 %}
 9669 
 9670 // This pattern is generated automatically from cas.m4.
 9671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9672 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9673   predicate(needs_acquiring_load_exclusive(n));
 9674   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9675   ins_cost(VOLATILE_REF_COST);
 9676   effect(TEMP_DEF res, KILL cr);
 9677   format %{
 9678     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9679   %}
 9680   ins_encode %{
 9681     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9682                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9683                /*weak*/ false, $res$$Register);
 9684   %}
 9685   ins_pipe(pipe_slow);
 9686 %}
 9687 
 9688 // This pattern is generated automatically from cas.m4.
 9689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9690 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9691   predicate(needs_acquiring_load_exclusive(n));
 9692   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9693   ins_cost(VOLATILE_REF_COST);
 9694   effect(TEMP_DEF res, KILL cr);
 9695   format %{
 9696     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9697   %}
 9698   ins_encode %{
 9699     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9700                Assembler::word, /*acquire*/ true, /*release*/ true,
 9701                /*weak*/ false, $res$$Register);
 9702   %}
 9703   ins_pipe(pipe_slow);
 9704 %}
 9705 
 9706 // This pattern is generated automatically from cas.m4.
 9707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9708 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9709   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9710   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9711   ins_cost(VOLATILE_REF_COST);
 9712   effect(TEMP_DEF res, KILL cr);
 9713   format %{
 9714     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9715   %}
 9716   ins_encode %{
 9717     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9718                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9719                /*weak*/ false, $res$$Register);
 9720   %}
 9721   ins_pipe(pipe_slow);
 9722 %}
 9723 
 9724 // This pattern is generated automatically from cas.m4.
 9725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9726 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9727 
 9728   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9729   ins_cost(2 * VOLATILE_REF_COST);
 9730   effect(KILL cr);
 9731   format %{
 9732     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9733     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9734   %}
 9735   ins_encode %{
 9736     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9737                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9738                /*weak*/ true, noreg);
 9739     __ csetw($res$$Register, Assembler::EQ);
 9740   %}
 9741   ins_pipe(pipe_slow);
 9742 %}
 9743 
 9744 // This pattern is generated automatically from cas.m4.
 9745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9746 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9747 
 9748   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9749   ins_cost(2 * VOLATILE_REF_COST);
 9750   effect(KILL cr);
 9751   format %{
 9752     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9753     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9754   %}
 9755   ins_encode %{
 9756     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9757                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9758                /*weak*/ true, noreg);
 9759     __ csetw($res$$Register, Assembler::EQ);
 9760   %}
 9761   ins_pipe(pipe_slow);
 9762 %}
 9763 
 9764 // This pattern is generated automatically from cas.m4.
 9765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9766 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9767 
 9768   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9769   ins_cost(2 * VOLATILE_REF_COST);
 9770   effect(KILL cr);
 9771   format %{
 9772     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9773     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9774   %}
 9775   ins_encode %{
 9776     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9777                Assembler::word, /*acquire*/ false, /*release*/ true,
 9778                /*weak*/ true, noreg);
 9779     __ csetw($res$$Register, Assembler::EQ);
 9780   %}
 9781   ins_pipe(pipe_slow);
 9782 %}
 9783 
 9784 // This pattern is generated automatically from cas.m4.
 9785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9786 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9787 
 9788   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9789   ins_cost(2 * VOLATILE_REF_COST);
 9790   effect(KILL cr);
 9791   format %{
 9792     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9793     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9794   %}
 9795   ins_encode %{
 9796     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9797                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9798                /*weak*/ true, noreg);
 9799     __ csetw($res$$Register, Assembler::EQ);
 9800   %}
 9801   ins_pipe(pipe_slow);
 9802 %}
 9803 
 9804 // This pattern is generated automatically from cas.m4.
 9805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9806 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9807 
 9808   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9809   ins_cost(2 * VOLATILE_REF_COST);
 9810   effect(KILL cr);
 9811   format %{
 9812     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9813     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9814   %}
 9815   ins_encode %{
 9816     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9817                Assembler::word, /*acquire*/ false, /*release*/ true,
 9818                /*weak*/ true, noreg);
 9819     __ csetw($res$$Register, Assembler::EQ);
 9820   %}
 9821   ins_pipe(pipe_slow);
 9822 %}
 9823 
 9824 // This pattern is generated automatically from cas.m4.
 9825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9826 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9827   predicate(n->as_LoadStore()->barrier_data() == 0);
 9828   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9829   ins_cost(2 * VOLATILE_REF_COST);
 9830   effect(KILL cr);
 9831   format %{
 9832     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9833     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9834   %}
 9835   ins_encode %{
 9836     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9837                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9838                /*weak*/ true, noreg);
 9839     __ csetw($res$$Register, Assembler::EQ);
 9840   %}
 9841   ins_pipe(pipe_slow);
 9842 %}
 9843 
 9844 // This pattern is generated automatically from cas.m4.
 9845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9846 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9847   predicate(needs_acquiring_load_exclusive(n));
 9848   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9849   ins_cost(VOLATILE_REF_COST);
 9850   effect(KILL cr);
 9851   format %{
 9852     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9853     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9854   %}
 9855   ins_encode %{
 9856     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9857                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9858                /*weak*/ true, noreg);
 9859     __ csetw($res$$Register, Assembler::EQ);
 9860   %}
 9861   ins_pipe(pipe_slow);
 9862 %}
 9863 
 9864 // This pattern is generated automatically from cas.m4.
 9865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9866 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9867   predicate(needs_acquiring_load_exclusive(n));
 9868   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9869   ins_cost(VOLATILE_REF_COST);
 9870   effect(KILL cr);
 9871   format %{
 9872     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9873     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9874   %}
 9875   ins_encode %{
 9876     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9877                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9878                /*weak*/ true, noreg);
 9879     __ csetw($res$$Register, Assembler::EQ);
 9880   %}
 9881   ins_pipe(pipe_slow);
 9882 %}
 9883 
 9884 // This pattern is generated automatically from cas.m4.
 9885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9886 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9887   predicate(needs_acquiring_load_exclusive(n));
 9888   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9889   ins_cost(VOLATILE_REF_COST);
 9890   effect(KILL cr);
 9891   format %{
 9892     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9893     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9894   %}
 9895   ins_encode %{
 9896     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9897                Assembler::word, /*acquire*/ true, /*release*/ true,
 9898                /*weak*/ true, noreg);
 9899     __ csetw($res$$Register, Assembler::EQ);
 9900   %}
 9901   ins_pipe(pipe_slow);
 9902 %}
 9903 
 9904 // This pattern is generated automatically from cas.m4.
 9905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9906 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9907   predicate(needs_acquiring_load_exclusive(n));
 9908   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9909   ins_cost(VOLATILE_REF_COST);
 9910   effect(KILL cr);
 9911   format %{
 9912     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9913     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9914   %}
 9915   ins_encode %{
 9916     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9917                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9918                /*weak*/ true, noreg);
 9919     __ csetw($res$$Register, Assembler::EQ);
 9920   %}
 9921   ins_pipe(pipe_slow);
 9922 %}
 9923 
 9924 // This pattern is generated automatically from cas.m4.
 9925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9926 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9927   predicate(needs_acquiring_load_exclusive(n));
 9928   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9929   ins_cost(VOLATILE_REF_COST);
 9930   effect(KILL cr);
 9931   format %{
 9932     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9933     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9934   %}
 9935   ins_encode %{
 9936     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9937                Assembler::word, /*acquire*/ true, /*release*/ true,
 9938                /*weak*/ true, noreg);
 9939     __ csetw($res$$Register, Assembler::EQ);
 9940   %}
 9941   ins_pipe(pipe_slow);
 9942 %}
 9943 
 9944 // This pattern is generated automatically from cas.m4.
 9945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9946 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9947   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9948   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9949   ins_cost(VOLATILE_REF_COST);
 9950   effect(KILL cr);
 9951   format %{
 9952     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9953     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9954   %}
 9955   ins_encode %{
 9956     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9957                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9958                /*weak*/ true, noreg);
 9959     __ csetw($res$$Register, Assembler::EQ);
 9960   %}
 9961   ins_pipe(pipe_slow);
 9962 %}
 9963 
 9964 // END This section of the file is automatically generated. Do not edit --------------
 9965 // ---------------------------------------------------------------------
 9966 
 9967 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9968   match(Set prev (GetAndSetI mem newv));
 9969   ins_cost(2 * VOLATILE_REF_COST);
 9970   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9971   ins_encode %{
 9972     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9973   %}
 9974   ins_pipe(pipe_serial);
 9975 %}
 9976 
 9977 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9978   match(Set prev (GetAndSetL mem newv));
 9979   ins_cost(2 * VOLATILE_REF_COST);
 9980   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9981   ins_encode %{
 9982     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9983   %}
 9984   ins_pipe(pipe_serial);
 9985 %}
 9986 
 9987 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9988   match(Set prev (GetAndSetN mem newv));
 9989   ins_cost(2 * VOLATILE_REF_COST);
 9990   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9991   ins_encode %{
 9992     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9993   %}
 9994   ins_pipe(pipe_serial);
 9995 %}
 9996 
 9997 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9998   predicate(n->as_LoadStore()->barrier_data() == 0);
 9999   match(Set prev (GetAndSetP mem newv));
10000   ins_cost(2 * VOLATILE_REF_COST);
10001   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
10002   ins_encode %{
10003     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
10004   %}
10005   ins_pipe(pipe_serial);
10006 %}
10007 
10008 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
10009   predicate(needs_acquiring_load_exclusive(n));
10010   match(Set prev (GetAndSetI mem newv));
10011   ins_cost(VOLATILE_REF_COST);
10012   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
10013   ins_encode %{
10014     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10015   %}
10016   ins_pipe(pipe_serial);
10017 %}
10018 
10019 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
10020   predicate(needs_acquiring_load_exclusive(n));
10021   match(Set prev (GetAndSetL mem newv));
10022   ins_cost(VOLATILE_REF_COST);
10023   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
10024   ins_encode %{
10025     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10026   %}
10027   ins_pipe(pipe_serial);
10028 %}
10029 
10030 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
10031   predicate(needs_acquiring_load_exclusive(n));
10032   match(Set prev (GetAndSetN mem newv));
10033   ins_cost(VOLATILE_REF_COST);
10034   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
10035   ins_encode %{
10036     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10037   %}
10038   ins_pipe(pipe_serial);
10039 %}
10040 
10041 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
10042   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
10043   match(Set prev (GetAndSetP mem newv));
10044   ins_cost(VOLATILE_REF_COST);
10045   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
10046   ins_encode %{
10047     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10048   %}
10049   ins_pipe(pipe_serial);
10050 %}
10051 
10052 
10053 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
10054   match(Set newval (GetAndAddL mem incr));
10055   ins_cost(2 * VOLATILE_REF_COST + 1);
10056   format %{ "get_and_addL $newval, [$mem], $incr" %}
10057   ins_encode %{
10058     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
10059   %}
10060   ins_pipe(pipe_serial);
10061 %}
10062 
10063 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
10064   predicate(n->as_LoadStore()->result_not_used());
10065   match(Set dummy (GetAndAddL mem incr));
10066   ins_cost(2 * VOLATILE_REF_COST);
10067   format %{ "get_and_addL [$mem], $incr" %}
10068   ins_encode %{
10069     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
10070   %}
10071   ins_pipe(pipe_serial);
10072 %}
10073 
10074 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10075   match(Set newval (GetAndAddL mem incr));
10076   ins_cost(2 * VOLATILE_REF_COST + 1);
10077   format %{ "get_and_addL $newval, [$mem], $incr" %}
10078   ins_encode %{
10079     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
10080   %}
10081   ins_pipe(pipe_serial);
10082 %}
10083 
10084 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
10085   predicate(n->as_LoadStore()->result_not_used());
10086   match(Set dummy (GetAndAddL mem incr));
10087   ins_cost(2 * VOLATILE_REF_COST);
10088   format %{ "get_and_addL [$mem], $incr" %}
10089   ins_encode %{
10090     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
10091   %}
10092   ins_pipe(pipe_serial);
10093 %}
10094 
10095 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10096   match(Set newval (GetAndAddI mem incr));
10097   ins_cost(2 * VOLATILE_REF_COST + 1);
10098   format %{ "get_and_addI $newval, [$mem], $incr" %}
10099   ins_encode %{
10100     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10101   %}
10102   ins_pipe(pipe_serial);
10103 %}
10104 
10105 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10106   predicate(n->as_LoadStore()->result_not_used());
10107   match(Set dummy (GetAndAddI mem incr));
10108   ins_cost(2 * VOLATILE_REF_COST);
10109   format %{ "get_and_addI [$mem], $incr" %}
10110   ins_encode %{
10111     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10112   %}
10113   ins_pipe(pipe_serial);
10114 %}
10115 
10116 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10117   match(Set newval (GetAndAddI mem incr));
10118   ins_cost(2 * VOLATILE_REF_COST + 1);
10119   format %{ "get_and_addI $newval, [$mem], $incr" %}
10120   ins_encode %{
10121     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10122   %}
10123   ins_pipe(pipe_serial);
10124 %}
10125 
10126 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10127   predicate(n->as_LoadStore()->result_not_used());
10128   match(Set dummy (GetAndAddI mem incr));
10129   ins_cost(2 * VOLATILE_REF_COST);
10130   format %{ "get_and_addI [$mem], $incr" %}
10131   ins_encode %{
10132     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10133   %}
10134   ins_pipe(pipe_serial);
10135 %}
10136 
10137 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10138   predicate(needs_acquiring_load_exclusive(n));
10139   match(Set newval (GetAndAddL mem incr));
10140   ins_cost(VOLATILE_REF_COST + 1);
10141   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10142   ins_encode %{
10143     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10144   %}
10145   ins_pipe(pipe_serial);
10146 %}
10147 
10148 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10149   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10150   match(Set dummy (GetAndAddL mem incr));
10151   ins_cost(VOLATILE_REF_COST);
10152   format %{ "get_and_addL_acq [$mem], $incr" %}
10153   ins_encode %{
10154     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10155   %}
10156   ins_pipe(pipe_serial);
10157 %}
10158 
10159 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10160   predicate(needs_acquiring_load_exclusive(n));
10161   match(Set newval (GetAndAddL mem incr));
10162   ins_cost(VOLATILE_REF_COST + 1);
10163   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10164   ins_encode %{
10165     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10166   %}
10167   ins_pipe(pipe_serial);
10168 %}
10169 
10170 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10171   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10172   match(Set dummy (GetAndAddL mem incr));
10173   ins_cost(VOLATILE_REF_COST);
10174   format %{ "get_and_addL_acq [$mem], $incr" %}
10175   ins_encode %{
10176     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10177   %}
10178   ins_pipe(pipe_serial);
10179 %}
10180 
10181 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10182   predicate(needs_acquiring_load_exclusive(n));
10183   match(Set newval (GetAndAddI mem incr));
10184   ins_cost(VOLATILE_REF_COST + 1);
10185   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10186   ins_encode %{
10187     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10188   %}
10189   ins_pipe(pipe_serial);
10190 %}
10191 
10192 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10193   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10194   match(Set dummy (GetAndAddI mem incr));
10195   ins_cost(VOLATILE_REF_COST);
10196   format %{ "get_and_addI_acq [$mem], $incr" %}
10197   ins_encode %{
10198     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10199   %}
10200   ins_pipe(pipe_serial);
10201 %}
10202 
10203 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10204   predicate(needs_acquiring_load_exclusive(n));
10205   match(Set newval (GetAndAddI mem incr));
10206   ins_cost(VOLATILE_REF_COST + 1);
10207   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10208   ins_encode %{
10209     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10210   %}
10211   ins_pipe(pipe_serial);
10212 %}
10213 
10214 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10215   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10216   match(Set dummy (GetAndAddI mem incr));
10217   ins_cost(VOLATILE_REF_COST);
10218   format %{ "get_and_addI_acq [$mem], $incr" %}
10219   ins_encode %{
10220     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10221   %}
10222   ins_pipe(pipe_serial);
10223 %}
10224 
10225 // Manifest a CmpL result in an integer register.
10226 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10227 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10228 %{
10229   match(Set dst (CmpL3 src1 src2));
10230   effect(KILL flags);
10231 
10232   ins_cost(INSN_COST * 6);
10233   format %{
10234       "cmp $src1, $src2"
10235       "csetw $dst, ne"
10236       "cnegw $dst, lt"
10237   %}
10238   // format %{ "CmpL3 $dst, $src1, $src2" %}
10239   ins_encode %{
10240     __ cmp($src1$$Register, $src2$$Register);
10241     __ csetw($dst$$Register, Assembler::NE);
10242     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10243   %}
10244 
10245   ins_pipe(pipe_class_default);
10246 %}
10247 
10248 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10249 %{
10250   match(Set dst (CmpL3 src1 src2));
10251   effect(KILL flags);
10252 
10253   ins_cost(INSN_COST * 6);
10254   format %{
10255       "cmp $src1, $src2"
10256       "csetw $dst, ne"
10257       "cnegw $dst, lt"
10258   %}
10259   ins_encode %{
10260     int32_t con = (int32_t)$src2$$constant;
10261      if (con < 0) {
10262       __ adds(zr, $src1$$Register, -con);
10263     } else {
10264       __ subs(zr, $src1$$Register, con);
10265     }
10266     __ csetw($dst$$Register, Assembler::NE);
10267     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10268   %}
10269 
10270   ins_pipe(pipe_class_default);
10271 %}
10272 
10273 // ============================================================================
10274 // Conditional Move Instructions
10275 
10276 // n.b. we have identical rules for both a signed compare op (cmpOp)
10277 // and an unsigned compare op (cmpOpU). it would be nice if we could
10278 // define an op class which merged both inputs and use it to type the
10279 // argument to a single rule. unfortunatelyt his fails because the
10280 // opclass does not live up to the COND_INTER interface of its
10281 // component operands. When the generic code tries to negate the
10282 // operand it ends up running the generci Machoper::negate method
10283 // which throws a ShouldNotHappen. So, we have to provide two flavours
10284 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10285 
10286 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10287   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10288 
10289   ins_cost(INSN_COST * 2);
10290   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10291 
10292   ins_encode %{
10293     __ cselw(as_Register($dst$$reg),
10294              as_Register($src2$$reg),
10295              as_Register($src1$$reg),
10296              (Assembler::Condition)$cmp$$cmpcode);
10297   %}
10298 
10299   ins_pipe(icond_reg_reg);
10300 %}
10301 
10302 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10303   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10304 
10305   ins_cost(INSN_COST * 2);
10306   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10307 
10308   ins_encode %{
10309     __ cselw(as_Register($dst$$reg),
10310              as_Register($src2$$reg),
10311              as_Register($src1$$reg),
10312              (Assembler::Condition)$cmp$$cmpcode);
10313   %}
10314 
10315   ins_pipe(icond_reg_reg);
10316 %}
10317 
10318 // special cases where one arg is zero
10319 
10320 // n.b. this is selected in preference to the rule above because it
10321 // avoids loading constant 0 into a source register
10322 
10323 // TODO
10324 // we ought only to be able to cull one of these variants as the ideal
10325 // transforms ought always to order the zero consistently (to left/right?)
10326 
10327 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10328   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10329 
10330   ins_cost(INSN_COST * 2);
10331   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10332 
10333   ins_encode %{
10334     __ cselw(as_Register($dst$$reg),
10335              as_Register($src$$reg),
10336              zr,
10337              (Assembler::Condition)$cmp$$cmpcode);
10338   %}
10339 
10340   ins_pipe(icond_reg);
10341 %}
10342 
10343 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10344   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10345 
10346   ins_cost(INSN_COST * 2);
10347   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10348 
10349   ins_encode %{
10350     __ cselw(as_Register($dst$$reg),
10351              as_Register($src$$reg),
10352              zr,
10353              (Assembler::Condition)$cmp$$cmpcode);
10354   %}
10355 
10356   ins_pipe(icond_reg);
10357 %}
10358 
10359 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10360   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10361 
10362   ins_cost(INSN_COST * 2);
10363   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10364 
10365   ins_encode %{
10366     __ cselw(as_Register($dst$$reg),
10367              zr,
10368              as_Register($src$$reg),
10369              (Assembler::Condition)$cmp$$cmpcode);
10370   %}
10371 
10372   ins_pipe(icond_reg);
10373 %}
10374 
10375 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10376   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10377 
10378   ins_cost(INSN_COST * 2);
10379   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10380 
10381   ins_encode %{
10382     __ cselw(as_Register($dst$$reg),
10383              zr,
10384              as_Register($src$$reg),
10385              (Assembler::Condition)$cmp$$cmpcode);
10386   %}
10387 
10388   ins_pipe(icond_reg);
10389 %}
10390 
10391 // special case for creating a boolean 0 or 1
10392 
10393 // n.b. this is selected in preference to the rule above because it
10394 // avoids loading constants 0 and 1 into a source register
10395 
10396 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10397   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10398 
10399   ins_cost(INSN_COST * 2);
10400   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10401 
10402   ins_encode %{
10403     // equivalently
10404     // cset(as_Register($dst$$reg),
10405     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10406     __ csincw(as_Register($dst$$reg),
10407              zr,
10408              zr,
10409              (Assembler::Condition)$cmp$$cmpcode);
10410   %}
10411 
10412   ins_pipe(icond_none);
10413 %}
10414 
10415 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10416   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10417 
10418   ins_cost(INSN_COST * 2);
10419   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10420 
10421   ins_encode %{
10422     // equivalently
10423     // cset(as_Register($dst$$reg),
10424     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10425     __ csincw(as_Register($dst$$reg),
10426              zr,
10427              zr,
10428              (Assembler::Condition)$cmp$$cmpcode);
10429   %}
10430 
10431   ins_pipe(icond_none);
10432 %}
10433 
10434 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10435   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10436 
10437   ins_cost(INSN_COST * 2);
10438   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10439 
10440   ins_encode %{
10441     __ csel(as_Register($dst$$reg),
10442             as_Register($src2$$reg),
10443             as_Register($src1$$reg),
10444             (Assembler::Condition)$cmp$$cmpcode);
10445   %}
10446 
10447   ins_pipe(icond_reg_reg);
10448 %}
10449 
10450 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10451   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10452 
10453   ins_cost(INSN_COST * 2);
10454   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10455 
10456   ins_encode %{
10457     __ csel(as_Register($dst$$reg),
10458             as_Register($src2$$reg),
10459             as_Register($src1$$reg),
10460             (Assembler::Condition)$cmp$$cmpcode);
10461   %}
10462 
10463   ins_pipe(icond_reg_reg);
10464 %}
10465 
10466 // special cases where one arg is zero
10467 
10468 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10469   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10470 
10471   ins_cost(INSN_COST * 2);
10472   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10473 
10474   ins_encode %{
10475     __ csel(as_Register($dst$$reg),
10476             zr,
10477             as_Register($src$$reg),
10478             (Assembler::Condition)$cmp$$cmpcode);
10479   %}
10480 
10481   ins_pipe(icond_reg);
10482 %}
10483 
10484 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10485   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10486 
10487   ins_cost(INSN_COST * 2);
10488   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10489 
10490   ins_encode %{
10491     __ csel(as_Register($dst$$reg),
10492             zr,
10493             as_Register($src$$reg),
10494             (Assembler::Condition)$cmp$$cmpcode);
10495   %}
10496 
10497   ins_pipe(icond_reg);
10498 %}
10499 
10500 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10501   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10502 
10503   ins_cost(INSN_COST * 2);
10504   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10505 
10506   ins_encode %{
10507     __ csel(as_Register($dst$$reg),
10508             as_Register($src$$reg),
10509             zr,
10510             (Assembler::Condition)$cmp$$cmpcode);
10511   %}
10512 
10513   ins_pipe(icond_reg);
10514 %}
10515 
10516 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10517   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10518 
10519   ins_cost(INSN_COST * 2);
10520   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10521 
10522   ins_encode %{
10523     __ csel(as_Register($dst$$reg),
10524             as_Register($src$$reg),
10525             zr,
10526             (Assembler::Condition)$cmp$$cmpcode);
10527   %}
10528 
10529   ins_pipe(icond_reg);
10530 %}
10531 
10532 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10533   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10534 
10535   ins_cost(INSN_COST * 2);
10536   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10537 
10538   ins_encode %{
10539     __ csel(as_Register($dst$$reg),
10540             as_Register($src2$$reg),
10541             as_Register($src1$$reg),
10542             (Assembler::Condition)$cmp$$cmpcode);
10543   %}
10544 
10545   ins_pipe(icond_reg_reg);
10546 %}
10547 
10548 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10549   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10550 
10551   ins_cost(INSN_COST * 2);
10552   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10553 
10554   ins_encode %{
10555     __ csel(as_Register($dst$$reg),
10556             as_Register($src2$$reg),
10557             as_Register($src1$$reg),
10558             (Assembler::Condition)$cmp$$cmpcode);
10559   %}
10560 
10561   ins_pipe(icond_reg_reg);
10562 %}
10563 
10564 // special cases where one arg is zero
10565 
10566 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10567   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10568 
10569   ins_cost(INSN_COST * 2);
10570   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10571 
10572   ins_encode %{
10573     __ csel(as_Register($dst$$reg),
10574             zr,
10575             as_Register($src$$reg),
10576             (Assembler::Condition)$cmp$$cmpcode);
10577   %}
10578 
10579   ins_pipe(icond_reg);
10580 %}
10581 
10582 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10583   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10584 
10585   ins_cost(INSN_COST * 2);
10586   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10587 
10588   ins_encode %{
10589     __ csel(as_Register($dst$$reg),
10590             zr,
10591             as_Register($src$$reg),
10592             (Assembler::Condition)$cmp$$cmpcode);
10593   %}
10594 
10595   ins_pipe(icond_reg);
10596 %}
10597 
10598 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10599   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10600 
10601   ins_cost(INSN_COST * 2);
10602   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10603 
10604   ins_encode %{
10605     __ csel(as_Register($dst$$reg),
10606             as_Register($src$$reg),
10607             zr,
10608             (Assembler::Condition)$cmp$$cmpcode);
10609   %}
10610 
10611   ins_pipe(icond_reg);
10612 %}
10613 
10614 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10615   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10616 
10617   ins_cost(INSN_COST * 2);
10618   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10619 
10620   ins_encode %{
10621     __ csel(as_Register($dst$$reg),
10622             as_Register($src$$reg),
10623             zr,
10624             (Assembler::Condition)$cmp$$cmpcode);
10625   %}
10626 
10627   ins_pipe(icond_reg);
10628 %}
10629 
10630 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10631   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10632 
10633   ins_cost(INSN_COST * 2);
10634   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10635 
10636   ins_encode %{
10637     __ cselw(as_Register($dst$$reg),
10638              as_Register($src2$$reg),
10639              as_Register($src1$$reg),
10640              (Assembler::Condition)$cmp$$cmpcode);
10641   %}
10642 
10643   ins_pipe(icond_reg_reg);
10644 %}
10645 
10646 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10647   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10648 
10649   ins_cost(INSN_COST * 2);
10650   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10651 
10652   ins_encode %{
10653     __ cselw(as_Register($dst$$reg),
10654              as_Register($src2$$reg),
10655              as_Register($src1$$reg),
10656              (Assembler::Condition)$cmp$$cmpcode);
10657   %}
10658 
10659   ins_pipe(icond_reg_reg);
10660 %}
10661 
10662 // special cases where one arg is zero
10663 
10664 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10665   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10666 
10667   ins_cost(INSN_COST * 2);
10668   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10669 
10670   ins_encode %{
10671     __ cselw(as_Register($dst$$reg),
10672              zr,
10673              as_Register($src$$reg),
10674              (Assembler::Condition)$cmp$$cmpcode);
10675   %}
10676 
10677   ins_pipe(icond_reg);
10678 %}
10679 
10680 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10681   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10682 
10683   ins_cost(INSN_COST * 2);
10684   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10685 
10686   ins_encode %{
10687     __ cselw(as_Register($dst$$reg),
10688              zr,
10689              as_Register($src$$reg),
10690              (Assembler::Condition)$cmp$$cmpcode);
10691   %}
10692 
10693   ins_pipe(icond_reg);
10694 %}
10695 
10696 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10697   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10698 
10699   ins_cost(INSN_COST * 2);
10700   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10701 
10702   ins_encode %{
10703     __ cselw(as_Register($dst$$reg),
10704              as_Register($src$$reg),
10705              zr,
10706              (Assembler::Condition)$cmp$$cmpcode);
10707   %}
10708 
10709   ins_pipe(icond_reg);
10710 %}
10711 
10712 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10713   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10714 
10715   ins_cost(INSN_COST * 2);
10716   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10717 
10718   ins_encode %{
10719     __ cselw(as_Register($dst$$reg),
10720              as_Register($src$$reg),
10721              zr,
10722              (Assembler::Condition)$cmp$$cmpcode);
10723   %}
10724 
10725   ins_pipe(icond_reg);
10726 %}
10727 
10728 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10729 %{
10730   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10731 
10732   ins_cost(INSN_COST * 3);
10733 
10734   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10735   ins_encode %{
10736     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10737     __ fcsels(as_FloatRegister($dst$$reg),
10738               as_FloatRegister($src2$$reg),
10739               as_FloatRegister($src1$$reg),
10740               cond);
10741   %}
10742 
10743   ins_pipe(fp_cond_reg_reg_s);
10744 %}
10745 
10746 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10747 %{
10748   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10749 
10750   ins_cost(INSN_COST * 3);
10751 
10752   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10753   ins_encode %{
10754     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10755     __ fcsels(as_FloatRegister($dst$$reg),
10756               as_FloatRegister($src2$$reg),
10757               as_FloatRegister($src1$$reg),
10758               cond);
10759   %}
10760 
10761   ins_pipe(fp_cond_reg_reg_s);
10762 %}
10763 
10764 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10765 %{
10766   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10767 
10768   ins_cost(INSN_COST * 3);
10769 
10770   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10771   ins_encode %{
10772     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10773     __ fcseld(as_FloatRegister($dst$$reg),
10774               as_FloatRegister($src2$$reg),
10775               as_FloatRegister($src1$$reg),
10776               cond);
10777   %}
10778 
10779   ins_pipe(fp_cond_reg_reg_d);
10780 %}
10781 
10782 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10783 %{
10784   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10785 
10786   ins_cost(INSN_COST * 3);
10787 
10788   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10789   ins_encode %{
10790     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10791     __ fcseld(as_FloatRegister($dst$$reg),
10792               as_FloatRegister($src2$$reg),
10793               as_FloatRegister($src1$$reg),
10794               cond);
10795   %}
10796 
10797   ins_pipe(fp_cond_reg_reg_d);
10798 %}
10799 
10800 // ============================================================================
10801 // Arithmetic Instructions
10802 //
10803 
10804 // Integer Addition
10805 
10806 // TODO
10807 // these currently employ operations which do not set CR and hence are
10808 // not flagged as killing CR but we would like to isolate the cases
10809 // where we want to set flags from those where we don't. need to work
10810 // out how to do that.
10811 
10812 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10813   match(Set dst (AddI src1 src2));
10814 
10815   ins_cost(INSN_COST);
10816   format %{ "addw  $dst, $src1, $src2" %}
10817 
10818   ins_encode %{
10819     __ addw(as_Register($dst$$reg),
10820             as_Register($src1$$reg),
10821             as_Register($src2$$reg));
10822   %}
10823 
10824   ins_pipe(ialu_reg_reg);
10825 %}
10826 
10827 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10828   match(Set dst (AddI src1 src2));
10829 
10830   ins_cost(INSN_COST);
10831   format %{ "addw $dst, $src1, $src2" %}
10832 
10833   // use opcode to indicate that this is an add not a sub
10834   opcode(0x0);
10835 
10836   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10837 
10838   ins_pipe(ialu_reg_imm);
10839 %}
10840 
10841 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10842   match(Set dst (AddI (ConvL2I src1) src2));
10843 
10844   ins_cost(INSN_COST);
10845   format %{ "addw $dst, $src1, $src2" %}
10846 
10847   // use opcode to indicate that this is an add not a sub
10848   opcode(0x0);
10849 
10850   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10851 
10852   ins_pipe(ialu_reg_imm);
10853 %}
10854 
10855 // Pointer Addition
10856 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10857   match(Set dst (AddP src1 src2));
10858 
10859   ins_cost(INSN_COST);
10860   format %{ "add $dst, $src1, $src2\t# ptr" %}
10861 
10862   ins_encode %{
10863     __ add(as_Register($dst$$reg),
10864            as_Register($src1$$reg),
10865            as_Register($src2$$reg));
10866   %}
10867 
10868   ins_pipe(ialu_reg_reg);
10869 %}
10870 
10871 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10872   match(Set dst (AddP src1 (ConvI2L src2)));
10873 
10874   ins_cost(1.9 * INSN_COST);
10875   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10876 
10877   ins_encode %{
10878     __ add(as_Register($dst$$reg),
10879            as_Register($src1$$reg),
10880            as_Register($src2$$reg), ext::sxtw);
10881   %}
10882 
10883   ins_pipe(ialu_reg_reg);
10884 %}
10885 
10886 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10887   match(Set dst (AddP src1 (LShiftL src2 scale)));
10888 
10889   ins_cost(1.9 * INSN_COST);
10890   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10891 
10892   ins_encode %{
10893     __ lea(as_Register($dst$$reg),
10894            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10895                    Address::lsl($scale$$constant)));
10896   %}
10897 
10898   ins_pipe(ialu_reg_reg_shift);
10899 %}
10900 
10901 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10902   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10903 
10904   ins_cost(1.9 * INSN_COST);
10905   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10906 
10907   ins_encode %{
10908     __ lea(as_Register($dst$$reg),
10909            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10910                    Address::sxtw($scale$$constant)));
10911   %}
10912 
10913   ins_pipe(ialu_reg_reg_shift);
10914 %}
10915 
10916 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10917   match(Set dst (LShiftL (ConvI2L src) scale));
10918 
10919   ins_cost(INSN_COST);
10920   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10921 
10922   ins_encode %{
10923     __ sbfiz(as_Register($dst$$reg),
10924           as_Register($src$$reg),
10925           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10926   %}
10927 
10928   ins_pipe(ialu_reg_shift);
10929 %}
10930 
10931 // Pointer Immediate Addition
10932 // n.b. this needs to be more expensive than using an indirect memory
10933 // operand
10934 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10935   match(Set dst (AddP src1 src2));
10936 
10937   ins_cost(INSN_COST);
10938   format %{ "add $dst, $src1, $src2\t# ptr" %}
10939 
10940   // use opcode to indicate that this is an add not a sub
10941   opcode(0x0);
10942 
10943   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10944 
10945   ins_pipe(ialu_reg_imm);
10946 %}
10947 
10948 // Long Addition
10949 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10950 
10951   match(Set dst (AddL src1 src2));
10952 
10953   ins_cost(INSN_COST);
10954   format %{ "add  $dst, $src1, $src2" %}
10955 
10956   ins_encode %{
10957     __ add(as_Register($dst$$reg),
10958            as_Register($src1$$reg),
10959            as_Register($src2$$reg));
10960   %}
10961 
10962   ins_pipe(ialu_reg_reg);
10963 %}
10964 
10965 // No constant pool entries requiredLong Immediate Addition.
10966 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10967   match(Set dst (AddL src1 src2));
10968 
10969   ins_cost(INSN_COST);
10970   format %{ "add $dst, $src1, $src2" %}
10971 
10972   // use opcode to indicate that this is an add not a sub
10973   opcode(0x0);
10974 
10975   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10976 
10977   ins_pipe(ialu_reg_imm);
10978 %}
10979 
10980 // Integer Subtraction
10981 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10982   match(Set dst (SubI src1 src2));
10983 
10984   ins_cost(INSN_COST);
10985   format %{ "subw  $dst, $src1, $src2" %}
10986 
10987   ins_encode %{
10988     __ subw(as_Register($dst$$reg),
10989             as_Register($src1$$reg),
10990             as_Register($src2$$reg));
10991   %}
10992 
10993   ins_pipe(ialu_reg_reg);
10994 %}
10995 
10996 // Immediate Subtraction
10997 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10998   match(Set dst (SubI src1 src2));
10999 
11000   ins_cost(INSN_COST);
11001   format %{ "subw $dst, $src1, $src2" %}
11002 
11003   // use opcode to indicate that this is a sub not an add
11004   opcode(0x1);
11005 
11006   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
11007 
11008   ins_pipe(ialu_reg_imm);
11009 %}
11010 
11011 // Long Subtraction
11012 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11013 
11014   match(Set dst (SubL src1 src2));
11015 
11016   ins_cost(INSN_COST);
11017   format %{ "sub  $dst, $src1, $src2" %}
11018 
11019   ins_encode %{
11020     __ sub(as_Register($dst$$reg),
11021            as_Register($src1$$reg),
11022            as_Register($src2$$reg));
11023   %}
11024 
11025   ins_pipe(ialu_reg_reg);
11026 %}
11027 
11028 // No constant pool entries requiredLong Immediate Subtraction.
11029 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
11030   match(Set dst (SubL src1 src2));
11031 
11032   ins_cost(INSN_COST);
11033   format %{ "sub$dst, $src1, $src2" %}
11034 
11035   // use opcode to indicate that this is a sub not an add
11036   opcode(0x1);
11037 
11038   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
11039 
11040   ins_pipe(ialu_reg_imm);
11041 %}
11042 
11043 // Integer Negation (special case for sub)
11044 
11045 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
11046   match(Set dst (SubI zero src));
11047 
11048   ins_cost(INSN_COST);
11049   format %{ "negw $dst, $src\t# int" %}
11050 
11051   ins_encode %{
11052     __ negw(as_Register($dst$$reg),
11053             as_Register($src$$reg));
11054   %}
11055 
11056   ins_pipe(ialu_reg);
11057 %}
11058 
11059 // Long Negation
11060 
11061 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
11062   match(Set dst (SubL zero src));
11063 
11064   ins_cost(INSN_COST);
11065   format %{ "neg $dst, $src\t# long" %}
11066 
11067   ins_encode %{
11068     __ neg(as_Register($dst$$reg),
11069            as_Register($src$$reg));
11070   %}
11071 
11072   ins_pipe(ialu_reg);
11073 %}
11074 
11075 // Integer Multiply
11076 
11077 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11078   match(Set dst (MulI src1 src2));
11079 
11080   ins_cost(INSN_COST * 3);
11081   format %{ "mulw  $dst, $src1, $src2" %}
11082 
11083   ins_encode %{
11084     __ mulw(as_Register($dst$$reg),
11085             as_Register($src1$$reg),
11086             as_Register($src2$$reg));
11087   %}
11088 
11089   ins_pipe(imul_reg_reg);
11090 %}
11091 
11092 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11093   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
11094 
11095   ins_cost(INSN_COST * 3);
11096   format %{ "smull  $dst, $src1, $src2" %}
11097 
11098   ins_encode %{
11099     __ smull(as_Register($dst$$reg),
11100              as_Register($src1$$reg),
11101              as_Register($src2$$reg));
11102   %}
11103 
11104   ins_pipe(imul_reg_reg);
11105 %}
11106 
11107 // Long Multiply
11108 
11109 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11110   match(Set dst (MulL src1 src2));
11111 
11112   ins_cost(INSN_COST * 5);
11113   format %{ "mul  $dst, $src1, $src2" %}
11114 
11115   ins_encode %{
11116     __ mul(as_Register($dst$$reg),
11117            as_Register($src1$$reg),
11118            as_Register($src2$$reg));
11119   %}
11120 
11121   ins_pipe(lmul_reg_reg);
11122 %}
11123 
11124 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11125 %{
11126   match(Set dst (MulHiL src1 src2));
11127 
11128   ins_cost(INSN_COST * 7);
11129   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
11130 
11131   ins_encode %{
11132     __ smulh(as_Register($dst$$reg),
11133              as_Register($src1$$reg),
11134              as_Register($src2$$reg));
11135   %}
11136 
11137   ins_pipe(lmul_reg_reg);
11138 %}
11139 
11140 // Combined Integer Multiply & Add/Sub
11141 
11142 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11143   match(Set dst (AddI src3 (MulI src1 src2)));
11144 
11145   ins_cost(INSN_COST * 3);
11146   format %{ "madd  $dst, $src1, $src2, $src3" %}
11147 
11148   ins_encode %{
11149     __ maddw(as_Register($dst$$reg),
11150              as_Register($src1$$reg),
11151              as_Register($src2$$reg),
11152              as_Register($src3$$reg));
11153   %}
11154 
11155   ins_pipe(imac_reg_reg);
11156 %}
11157 
11158 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11159   match(Set dst (SubI src3 (MulI src1 src2)));
11160 
11161   ins_cost(INSN_COST * 3);
11162   format %{ "msub  $dst, $src1, $src2, $src3" %}
11163 
11164   ins_encode %{
11165     __ msubw(as_Register($dst$$reg),
11166              as_Register($src1$$reg),
11167              as_Register($src2$$reg),
11168              as_Register($src3$$reg));
11169   %}
11170 
11171   ins_pipe(imac_reg_reg);
11172 %}
11173 
11174 // Combined Integer Multiply & Neg
11175 
11176 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11177   match(Set dst (MulI (SubI zero src1) src2));
11178 
11179   ins_cost(INSN_COST * 3);
11180   format %{ "mneg  $dst, $src1, $src2" %}
11181 
11182   ins_encode %{
11183     __ mnegw(as_Register($dst$$reg),
11184              as_Register($src1$$reg),
11185              as_Register($src2$$reg));
11186   %}
11187 
11188   ins_pipe(imac_reg_reg);
11189 %}
11190 
11191 // Combined Long Multiply & Add/Sub
11192 
11193 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11194   match(Set dst (AddL src3 (MulL src1 src2)));
11195 
11196   ins_cost(INSN_COST * 5);
11197   format %{ "madd  $dst, $src1, $src2, $src3" %}
11198 
11199   ins_encode %{
11200     __ madd(as_Register($dst$$reg),
11201             as_Register($src1$$reg),
11202             as_Register($src2$$reg),
11203             as_Register($src3$$reg));
11204   %}
11205 
11206   ins_pipe(lmac_reg_reg);
11207 %}
11208 
11209 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11210   match(Set dst (SubL src3 (MulL src1 src2)));
11211 
11212   ins_cost(INSN_COST * 5);
11213   format %{ "msub  $dst, $src1, $src2, $src3" %}
11214 
11215   ins_encode %{
11216     __ msub(as_Register($dst$$reg),
11217             as_Register($src1$$reg),
11218             as_Register($src2$$reg),
11219             as_Register($src3$$reg));
11220   %}
11221 
11222   ins_pipe(lmac_reg_reg);
11223 %}
11224 
11225 // Combined Long Multiply & Neg
11226 
11227 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11228   match(Set dst (MulL (SubL zero src1) src2));
11229 
11230   ins_cost(INSN_COST * 5);
11231   format %{ "mneg  $dst, $src1, $src2" %}
11232 
11233   ins_encode %{
11234     __ mneg(as_Register($dst$$reg),
11235             as_Register($src1$$reg),
11236             as_Register($src2$$reg));
11237   %}
11238 
11239   ins_pipe(lmac_reg_reg);
11240 %}
11241 
11242 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11243 
11244 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11245   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11246 
11247   ins_cost(INSN_COST * 3);
11248   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11249 
11250   ins_encode %{
11251     __ smaddl(as_Register($dst$$reg),
11252               as_Register($src1$$reg),
11253               as_Register($src2$$reg),
11254               as_Register($src3$$reg));
11255   %}
11256 
11257   ins_pipe(imac_reg_reg);
11258 %}
11259 
11260 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11261   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11262 
11263   ins_cost(INSN_COST * 3);
11264   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11265 
11266   ins_encode %{
11267     __ smsubl(as_Register($dst$$reg),
11268               as_Register($src1$$reg),
11269               as_Register($src2$$reg),
11270               as_Register($src3$$reg));
11271   %}
11272 
11273   ins_pipe(imac_reg_reg);
11274 %}
11275 
11276 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11277   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11278 
11279   ins_cost(INSN_COST * 3);
11280   format %{ "smnegl  $dst, $src1, $src2" %}
11281 
11282   ins_encode %{
11283     __ smnegl(as_Register($dst$$reg),
11284               as_Register($src1$$reg),
11285               as_Register($src2$$reg));
11286   %}
11287 
11288   ins_pipe(imac_reg_reg);
11289 %}
11290 
11291 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11292 
11293 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11294   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11295 
11296   ins_cost(INSN_COST * 5);
11297   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11298             "maddw $dst, $src3, $src4, rscratch1" %}
11299 
11300   ins_encode %{
11301     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11302     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11303 
11304   ins_pipe(imac_reg_reg);
11305 %}
11306 
11307 // Integer Divide
11308 
11309 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11310   match(Set dst (DivI src1 src2));
11311 
11312   ins_cost(INSN_COST * 19);
11313   format %{ "sdivw  $dst, $src1, $src2" %}
11314 
11315   ins_encode(aarch64_enc_divw(dst, src1, src2));
11316   ins_pipe(idiv_reg_reg);
11317 %}
11318 
11319 // Long Divide
11320 
11321 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11322   match(Set dst (DivL src1 src2));
11323 
11324   ins_cost(INSN_COST * 35);
11325   format %{ "sdiv   $dst, $src1, $src2" %}
11326 
11327   ins_encode(aarch64_enc_div(dst, src1, src2));
11328   ins_pipe(ldiv_reg_reg);
11329 %}
11330 
11331 // Integer Remainder
11332 
11333 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11334   match(Set dst (ModI src1 src2));
11335 
11336   ins_cost(INSN_COST * 22);
11337   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11338             "msubw($dst, rscratch1, $src2, $src1" %}
11339 
11340   ins_encode(aarch64_enc_modw(dst, src1, src2));
11341   ins_pipe(idiv_reg_reg);
11342 %}
11343 
11344 // Long Remainder
11345 
11346 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11347   match(Set dst (ModL src1 src2));
11348 
11349   ins_cost(INSN_COST * 38);
11350   format %{ "sdiv   rscratch1, $src1, $src2\n"
11351             "msub($dst, rscratch1, $src2, $src1" %}
11352 
11353   ins_encode(aarch64_enc_mod(dst, src1, src2));
11354   ins_pipe(ldiv_reg_reg);
11355 %}
11356 
11357 // Integer Shifts
11358 
11359 // Shift Left Register
11360 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11361   match(Set dst (LShiftI src1 src2));
11362 
11363   ins_cost(INSN_COST * 2);
11364   format %{ "lslvw  $dst, $src1, $src2" %}
11365 
11366   ins_encode %{
11367     __ lslvw(as_Register($dst$$reg),
11368              as_Register($src1$$reg),
11369              as_Register($src2$$reg));
11370   %}
11371 
11372   ins_pipe(ialu_reg_reg_vshift);
11373 %}
11374 
11375 // Shift Left Immediate
11376 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11377   match(Set dst (LShiftI src1 src2));
11378 
11379   ins_cost(INSN_COST);
11380   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11381 
11382   ins_encode %{
11383     __ lslw(as_Register($dst$$reg),
11384             as_Register($src1$$reg),
11385             $src2$$constant & 0x1f);
11386   %}
11387 
11388   ins_pipe(ialu_reg_shift);
11389 %}
11390 
11391 // Shift Right Logical Register
11392 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11393   match(Set dst (URShiftI src1 src2));
11394 
11395   ins_cost(INSN_COST * 2);
11396   format %{ "lsrvw  $dst, $src1, $src2" %}
11397 
11398   ins_encode %{
11399     __ lsrvw(as_Register($dst$$reg),
11400              as_Register($src1$$reg),
11401              as_Register($src2$$reg));
11402   %}
11403 
11404   ins_pipe(ialu_reg_reg_vshift);
11405 %}
11406 
11407 // Shift Right Logical Immediate
11408 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11409   match(Set dst (URShiftI src1 src2));
11410 
11411   ins_cost(INSN_COST);
11412   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11413 
11414   ins_encode %{
11415     __ lsrw(as_Register($dst$$reg),
11416             as_Register($src1$$reg),
11417             $src2$$constant & 0x1f);
11418   %}
11419 
11420   ins_pipe(ialu_reg_shift);
11421 %}
11422 
11423 // Shift Right Arithmetic Register
11424 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11425   match(Set dst (RShiftI src1 src2));
11426 
11427   ins_cost(INSN_COST * 2);
11428   format %{ "asrvw  $dst, $src1, $src2" %}
11429 
11430   ins_encode %{
11431     __ asrvw(as_Register($dst$$reg),
11432              as_Register($src1$$reg),
11433              as_Register($src2$$reg));
11434   %}
11435 
11436   ins_pipe(ialu_reg_reg_vshift);
11437 %}
11438 
11439 // Shift Right Arithmetic Immediate
11440 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11441   match(Set dst (RShiftI src1 src2));
11442 
11443   ins_cost(INSN_COST);
11444   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11445 
11446   ins_encode %{
11447     __ asrw(as_Register($dst$$reg),
11448             as_Register($src1$$reg),
11449             $src2$$constant & 0x1f);
11450   %}
11451 
11452   ins_pipe(ialu_reg_shift);
11453 %}
11454 
11455 // Combined Int Mask and Right Shift (using UBFM)
11456 // TODO
11457 
11458 // Long Shifts
11459 
11460 // Shift Left Register
11461 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11462   match(Set dst (LShiftL src1 src2));
11463 
11464   ins_cost(INSN_COST * 2);
11465   format %{ "lslv  $dst, $src1, $src2" %}
11466 
11467   ins_encode %{
11468     __ lslv(as_Register($dst$$reg),
11469             as_Register($src1$$reg),
11470             as_Register($src2$$reg));
11471   %}
11472 
11473   ins_pipe(ialu_reg_reg_vshift);
11474 %}
11475 
11476 // Shift Left Immediate
11477 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11478   match(Set dst (LShiftL src1 src2));
11479 
11480   ins_cost(INSN_COST);
11481   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11482 
11483   ins_encode %{
11484     __ lsl(as_Register($dst$$reg),
11485             as_Register($src1$$reg),
11486             $src2$$constant & 0x3f);
11487   %}
11488 
11489   ins_pipe(ialu_reg_shift);
11490 %}
11491 
11492 // Shift Right Logical Register
11493 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11494   match(Set dst (URShiftL src1 src2));
11495 
11496   ins_cost(INSN_COST * 2);
11497   format %{ "lsrv  $dst, $src1, $src2" %}
11498 
11499   ins_encode %{
11500     __ lsrv(as_Register($dst$$reg),
11501             as_Register($src1$$reg),
11502             as_Register($src2$$reg));
11503   %}
11504 
11505   ins_pipe(ialu_reg_reg_vshift);
11506 %}
11507 
11508 // Shift Right Logical Immediate
11509 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11510   match(Set dst (URShiftL src1 src2));
11511 
11512   ins_cost(INSN_COST);
11513   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11514 
11515   ins_encode %{
11516     __ lsr(as_Register($dst$$reg),
11517            as_Register($src1$$reg),
11518            $src2$$constant & 0x3f);
11519   %}
11520 
11521   ins_pipe(ialu_reg_shift);
11522 %}
11523 
11524 // A special-case pattern for card table stores.
11525 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11526   match(Set dst (URShiftL (CastP2X src1) src2));
11527 
11528   ins_cost(INSN_COST);
11529   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11530 
11531   ins_encode %{
11532     __ lsr(as_Register($dst$$reg),
11533            as_Register($src1$$reg),
11534            $src2$$constant & 0x3f);
11535   %}
11536 
11537   ins_pipe(ialu_reg_shift);
11538 %}
11539 
11540 // Shift Right Arithmetic Register
11541 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11542   match(Set dst (RShiftL src1 src2));
11543 
11544   ins_cost(INSN_COST * 2);
11545   format %{ "asrv  $dst, $src1, $src2" %}
11546 
11547   ins_encode %{
11548     __ asrv(as_Register($dst$$reg),
11549             as_Register($src1$$reg),
11550             as_Register($src2$$reg));
11551   %}
11552 
11553   ins_pipe(ialu_reg_reg_vshift);
11554 %}
11555 
11556 // Shift Right Arithmetic Immediate
11557 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11558   match(Set dst (RShiftL src1 src2));
11559 
11560   ins_cost(INSN_COST);
11561   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11562 
11563   ins_encode %{
11564     __ asr(as_Register($dst$$reg),
11565            as_Register($src1$$reg),
11566            $src2$$constant & 0x3f);
11567   %}
11568 
11569   ins_pipe(ialu_reg_shift);
11570 %}
11571 
11572 // BEGIN This section of the file is automatically generated. Do not edit --------------
11573 // This section is generated from aarch64_ad.m4
11574 
11575 
11576 // This pattern is automatically generated from aarch64_ad.m4
11577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11578 instruct regL_not_reg(iRegLNoSp dst,
11579                          iRegL src1, immL_M1 m1,
11580                          rFlagsReg cr) %{
11581   match(Set dst (XorL src1 m1));
11582   ins_cost(INSN_COST);
11583   format %{ "eon  $dst, $src1, zr" %}
11584 
11585   ins_encode %{
11586     __ eon(as_Register($dst$$reg),
11587               as_Register($src1$$reg),
11588               zr,
11589               Assembler::LSL, 0);
11590   %}
11591 
11592   ins_pipe(ialu_reg);
11593 %}
11594 
11595 // This pattern is automatically generated from aarch64_ad.m4
11596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11597 instruct regI_not_reg(iRegINoSp dst,
11598                          iRegIorL2I src1, immI_M1 m1,
11599                          rFlagsReg cr) %{
11600   match(Set dst (XorI src1 m1));
11601   ins_cost(INSN_COST);
11602   format %{ "eonw  $dst, $src1, zr" %}
11603 
11604   ins_encode %{
11605     __ eonw(as_Register($dst$$reg),
11606               as_Register($src1$$reg),
11607               zr,
11608               Assembler::LSL, 0);
11609   %}
11610 
11611   ins_pipe(ialu_reg);
11612 %}
11613 
11614 // This pattern is automatically generated from aarch64_ad.m4
11615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11616 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11617                               immI0 zero, iRegIorL2I src1, immI src2) %{
11618   match(Set dst (SubI zero (URShiftI src1 src2)));
11619 
11620   ins_cost(1.9 * INSN_COST);
11621   format %{ "negw  $dst, $src1, LSR $src2" %}
11622 
11623   ins_encode %{
11624     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11625             Assembler::LSR, $src2$$constant & 0x1f);
11626   %}
11627 
11628   ins_pipe(ialu_reg_shift);
11629 %}
11630 
11631 // This pattern is automatically generated from aarch64_ad.m4
11632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11633 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11634                               immI0 zero, iRegIorL2I src1, immI src2) %{
11635   match(Set dst (SubI zero (RShiftI src1 src2)));
11636 
11637   ins_cost(1.9 * INSN_COST);
11638   format %{ "negw  $dst, $src1, ASR $src2" %}
11639 
11640   ins_encode %{
11641     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11642             Assembler::ASR, $src2$$constant & 0x1f);
11643   %}
11644 
11645   ins_pipe(ialu_reg_shift);
11646 %}
11647 
11648 // This pattern is automatically generated from aarch64_ad.m4
11649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11650 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11651                               immI0 zero, iRegIorL2I src1, immI src2) %{
11652   match(Set dst (SubI zero (LShiftI src1 src2)));
11653 
11654   ins_cost(1.9 * INSN_COST);
11655   format %{ "negw  $dst, $src1, LSL $src2" %}
11656 
11657   ins_encode %{
11658     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11659             Assembler::LSL, $src2$$constant & 0x1f);
11660   %}
11661 
11662   ins_pipe(ialu_reg_shift);
11663 %}
11664 
11665 // This pattern is automatically generated from aarch64_ad.m4
11666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11667 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11668                               immL0 zero, iRegL src1, immI src2) %{
11669   match(Set dst (SubL zero (URShiftL src1 src2)));
11670 
11671   ins_cost(1.9 * INSN_COST);
11672   format %{ "neg  $dst, $src1, LSR $src2" %}
11673 
11674   ins_encode %{
11675     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11676             Assembler::LSR, $src2$$constant & 0x3f);
11677   %}
11678 
11679   ins_pipe(ialu_reg_shift);
11680 %}
11681 
11682 // This pattern is automatically generated from aarch64_ad.m4
11683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11684 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11685                               immL0 zero, iRegL src1, immI src2) %{
11686   match(Set dst (SubL zero (RShiftL src1 src2)));
11687 
11688   ins_cost(1.9 * INSN_COST);
11689   format %{ "neg  $dst, $src1, ASR $src2" %}
11690 
11691   ins_encode %{
11692     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11693             Assembler::ASR, $src2$$constant & 0x3f);
11694   %}
11695 
11696   ins_pipe(ialu_reg_shift);
11697 %}
11698 
11699 // This pattern is automatically generated from aarch64_ad.m4
11700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11701 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11702                               immL0 zero, iRegL src1, immI src2) %{
11703   match(Set dst (SubL zero (LShiftL src1 src2)));
11704 
11705   ins_cost(1.9 * INSN_COST);
11706   format %{ "neg  $dst, $src1, LSL $src2" %}
11707 
11708   ins_encode %{
11709     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11710             Assembler::LSL, $src2$$constant & 0x3f);
11711   %}
11712 
11713   ins_pipe(ialu_reg_shift);
11714 %}
11715 
11716 // This pattern is automatically generated from aarch64_ad.m4
11717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11718 instruct AndI_reg_not_reg(iRegINoSp dst,
11719                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11720   match(Set dst (AndI src1 (XorI src2 m1)));
11721   ins_cost(INSN_COST);
11722   format %{ "bicw  $dst, $src1, $src2" %}
11723 
11724   ins_encode %{
11725     __ bicw(as_Register($dst$$reg),
11726               as_Register($src1$$reg),
11727               as_Register($src2$$reg),
11728               Assembler::LSL, 0);
11729   %}
11730 
11731   ins_pipe(ialu_reg_reg);
11732 %}
11733 
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 instruct AndL_reg_not_reg(iRegLNoSp dst,
11737                          iRegL src1, iRegL src2, immL_M1 m1) %{
11738   match(Set dst (AndL src1 (XorL src2 m1)));
11739   ins_cost(INSN_COST);
11740   format %{ "bic  $dst, $src1, $src2" %}
11741 
11742   ins_encode %{
11743     __ bic(as_Register($dst$$reg),
11744               as_Register($src1$$reg),
11745               as_Register($src2$$reg),
11746               Assembler::LSL, 0);
11747   %}
11748 
11749   ins_pipe(ialu_reg_reg);
11750 %}
11751 
11752 // This pattern is automatically generated from aarch64_ad.m4
11753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11754 instruct OrI_reg_not_reg(iRegINoSp dst,
11755                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11756   match(Set dst (OrI src1 (XorI src2 m1)));
11757   ins_cost(INSN_COST);
11758   format %{ "ornw  $dst, $src1, $src2" %}
11759 
11760   ins_encode %{
11761     __ ornw(as_Register($dst$$reg),
11762               as_Register($src1$$reg),
11763               as_Register($src2$$reg),
11764               Assembler::LSL, 0);
11765   %}
11766 
11767   ins_pipe(ialu_reg_reg);
11768 %}
11769 
11770 // This pattern is automatically generated from aarch64_ad.m4
11771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11772 instruct OrL_reg_not_reg(iRegLNoSp dst,
11773                          iRegL src1, iRegL src2, immL_M1 m1) %{
11774   match(Set dst (OrL src1 (XorL src2 m1)));
11775   ins_cost(INSN_COST);
11776   format %{ "orn  $dst, $src1, $src2" %}
11777 
11778   ins_encode %{
11779     __ orn(as_Register($dst$$reg),
11780               as_Register($src1$$reg),
11781               as_Register($src2$$reg),
11782               Assembler::LSL, 0);
11783   %}
11784 
11785   ins_pipe(ialu_reg_reg);
11786 %}
11787 
11788 // This pattern is automatically generated from aarch64_ad.m4
11789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11790 instruct XorI_reg_not_reg(iRegINoSp dst,
11791                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11792   match(Set dst (XorI m1 (XorI src2 src1)));
11793   ins_cost(INSN_COST);
11794   format %{ "eonw  $dst, $src1, $src2" %}
11795 
11796   ins_encode %{
11797     __ eonw(as_Register($dst$$reg),
11798               as_Register($src1$$reg),
11799               as_Register($src2$$reg),
11800               Assembler::LSL, 0);
11801   %}
11802 
11803   ins_pipe(ialu_reg_reg);
11804 %}
11805 
11806 // This pattern is automatically generated from aarch64_ad.m4
11807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11808 instruct XorL_reg_not_reg(iRegLNoSp dst,
11809                          iRegL src1, iRegL src2, immL_M1 m1) %{
11810   match(Set dst (XorL m1 (XorL src2 src1)));
11811   ins_cost(INSN_COST);
11812   format %{ "eon  $dst, $src1, $src2" %}
11813 
11814   ins_encode %{
11815     __ eon(as_Register($dst$$reg),
11816               as_Register($src1$$reg),
11817               as_Register($src2$$reg),
11818               Assembler::LSL, 0);
11819   %}
11820 
11821   ins_pipe(ialu_reg_reg);
11822 %}
11823 
11824 // This pattern is automatically generated from aarch64_ad.m4
11825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11826 // val & (-1 ^ (val >>> shift)) ==> bicw
11827 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11828                          iRegIorL2I src1, iRegIorL2I src2,
11829                          immI src3, immI_M1 src4) %{
11830   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11831   ins_cost(1.9 * INSN_COST);
11832   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11833 
11834   ins_encode %{
11835     __ bicw(as_Register($dst$$reg),
11836               as_Register($src1$$reg),
11837               as_Register($src2$$reg),
11838               Assembler::LSR,
11839               $src3$$constant & 0x1f);
11840   %}
11841 
11842   ins_pipe(ialu_reg_reg_shift);
11843 %}
11844 
11845 // This pattern is automatically generated from aarch64_ad.m4
11846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11847 // val & (-1 ^ (val >>> shift)) ==> bic
11848 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11849                          iRegL src1, iRegL src2,
11850                          immI src3, immL_M1 src4) %{
11851   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11852   ins_cost(1.9 * INSN_COST);
11853   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11854 
11855   ins_encode %{
11856     __ bic(as_Register($dst$$reg),
11857               as_Register($src1$$reg),
11858               as_Register($src2$$reg),
11859               Assembler::LSR,
11860               $src3$$constant & 0x3f);
11861   %}
11862 
11863   ins_pipe(ialu_reg_reg_shift);
11864 %}
11865 
11866 // This pattern is automatically generated from aarch64_ad.m4
11867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11868 // val & (-1 ^ (val >> shift)) ==> bicw
11869 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11870                          iRegIorL2I src1, iRegIorL2I src2,
11871                          immI src3, immI_M1 src4) %{
11872   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11873   ins_cost(1.9 * INSN_COST);
11874   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11875 
11876   ins_encode %{
11877     __ bicw(as_Register($dst$$reg),
11878               as_Register($src1$$reg),
11879               as_Register($src2$$reg),
11880               Assembler::ASR,
11881               $src3$$constant & 0x1f);
11882   %}
11883 
11884   ins_pipe(ialu_reg_reg_shift);
11885 %}
11886 
11887 // This pattern is automatically generated from aarch64_ad.m4
11888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11889 // val & (-1 ^ (val >> shift)) ==> bic
11890 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11891                          iRegL src1, iRegL src2,
11892                          immI src3, immL_M1 src4) %{
11893   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11894   ins_cost(1.9 * INSN_COST);
11895   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11896 
11897   ins_encode %{
11898     __ bic(as_Register($dst$$reg),
11899               as_Register($src1$$reg),
11900               as_Register($src2$$reg),
11901               Assembler::ASR,
11902               $src3$$constant & 0x3f);
11903   %}
11904 
11905   ins_pipe(ialu_reg_reg_shift);
11906 %}
11907 
11908 // This pattern is automatically generated from aarch64_ad.m4
11909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11910 // val & (-1 ^ (val ror shift)) ==> bicw
11911 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11912                          iRegIorL2I src1, iRegIorL2I src2,
11913                          immI src3, immI_M1 src4) %{
11914   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11915   ins_cost(1.9 * INSN_COST);
11916   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11917 
11918   ins_encode %{
11919     __ bicw(as_Register($dst$$reg),
11920               as_Register($src1$$reg),
11921               as_Register($src2$$reg),
11922               Assembler::ROR,
11923               $src3$$constant & 0x1f);
11924   %}
11925 
11926   ins_pipe(ialu_reg_reg_shift);
11927 %}
11928 
11929 // This pattern is automatically generated from aarch64_ad.m4
11930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11931 // val & (-1 ^ (val ror shift)) ==> bic
11932 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11933                          iRegL src1, iRegL src2,
11934                          immI src3, immL_M1 src4) %{
11935   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11936   ins_cost(1.9 * INSN_COST);
11937   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11938 
11939   ins_encode %{
11940     __ bic(as_Register($dst$$reg),
11941               as_Register($src1$$reg),
11942               as_Register($src2$$reg),
11943               Assembler::ROR,
11944               $src3$$constant & 0x3f);
11945   %}
11946 
11947   ins_pipe(ialu_reg_reg_shift);
11948 %}
11949 
11950 // This pattern is automatically generated from aarch64_ad.m4
11951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11952 // val & (-1 ^ (val << shift)) ==> bicw
11953 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11954                          iRegIorL2I src1, iRegIorL2I src2,
11955                          immI src3, immI_M1 src4) %{
11956   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11957   ins_cost(1.9 * INSN_COST);
11958   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11959 
11960   ins_encode %{
11961     __ bicw(as_Register($dst$$reg),
11962               as_Register($src1$$reg),
11963               as_Register($src2$$reg),
11964               Assembler::LSL,
11965               $src3$$constant & 0x1f);
11966   %}
11967 
11968   ins_pipe(ialu_reg_reg_shift);
11969 %}
11970 
11971 // This pattern is automatically generated from aarch64_ad.m4
11972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11973 // val & (-1 ^ (val << shift)) ==> bic
11974 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11975                          iRegL src1, iRegL src2,
11976                          immI src3, immL_M1 src4) %{
11977   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11978   ins_cost(1.9 * INSN_COST);
11979   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11980 
11981   ins_encode %{
11982     __ bic(as_Register($dst$$reg),
11983               as_Register($src1$$reg),
11984               as_Register($src2$$reg),
11985               Assembler::LSL,
11986               $src3$$constant & 0x3f);
11987   %}
11988 
11989   ins_pipe(ialu_reg_reg_shift);
11990 %}
11991 
11992 // This pattern is automatically generated from aarch64_ad.m4
11993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11994 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11995 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11996                          iRegIorL2I src1, iRegIorL2I src2,
11997                          immI src3, immI_M1 src4) %{
11998   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11999   ins_cost(1.9 * INSN_COST);
12000   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
12001 
12002   ins_encode %{
12003     __ eonw(as_Register($dst$$reg),
12004               as_Register($src1$$reg),
12005               as_Register($src2$$reg),
12006               Assembler::LSR,
12007               $src3$$constant & 0x1f);
12008   %}
12009 
12010   ins_pipe(ialu_reg_reg_shift);
12011 %}
12012 
12013 // This pattern is automatically generated from aarch64_ad.m4
12014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12015 // val ^ (-1 ^ (val >>> shift)) ==> eon
12016 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
12017                          iRegL src1, iRegL src2,
12018                          immI src3, immL_M1 src4) %{
12019   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
12020   ins_cost(1.9 * INSN_COST);
12021   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
12022 
12023   ins_encode %{
12024     __ eon(as_Register($dst$$reg),
12025               as_Register($src1$$reg),
12026               as_Register($src2$$reg),
12027               Assembler::LSR,
12028               $src3$$constant & 0x3f);
12029   %}
12030 
12031   ins_pipe(ialu_reg_reg_shift);
12032 %}
12033 
12034 // This pattern is automatically generated from aarch64_ad.m4
12035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12036 // val ^ (-1 ^ (val >> shift)) ==> eonw
12037 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
12038                          iRegIorL2I src1, iRegIorL2I src2,
12039                          immI src3, immI_M1 src4) %{
12040   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
12041   ins_cost(1.9 * INSN_COST);
12042   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
12043 
12044   ins_encode %{
12045     __ eonw(as_Register($dst$$reg),
12046               as_Register($src1$$reg),
12047               as_Register($src2$$reg),
12048               Assembler::ASR,
12049               $src3$$constant & 0x1f);
12050   %}
12051 
12052   ins_pipe(ialu_reg_reg_shift);
12053 %}
12054 
12055 // This pattern is automatically generated from aarch64_ad.m4
12056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12057 // val ^ (-1 ^ (val >> shift)) ==> eon
12058 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
12059                          iRegL src1, iRegL src2,
12060                          immI src3, immL_M1 src4) %{
12061   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
12062   ins_cost(1.9 * INSN_COST);
12063   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
12064 
12065   ins_encode %{
12066     __ eon(as_Register($dst$$reg),
12067               as_Register($src1$$reg),
12068               as_Register($src2$$reg),
12069               Assembler::ASR,
12070               $src3$$constant & 0x3f);
12071   %}
12072 
12073   ins_pipe(ialu_reg_reg_shift);
12074 %}
12075 
12076 // This pattern is automatically generated from aarch64_ad.m4
12077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12078 // val ^ (-1 ^ (val ror shift)) ==> eonw
12079 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
12080                          iRegIorL2I src1, iRegIorL2I src2,
12081                          immI src3, immI_M1 src4) %{
12082   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
12083   ins_cost(1.9 * INSN_COST);
12084   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
12085 
12086   ins_encode %{
12087     __ eonw(as_Register($dst$$reg),
12088               as_Register($src1$$reg),
12089               as_Register($src2$$reg),
12090               Assembler::ROR,
12091               $src3$$constant & 0x1f);
12092   %}
12093 
12094   ins_pipe(ialu_reg_reg_shift);
12095 %}
12096 
12097 // This pattern is automatically generated from aarch64_ad.m4
12098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12099 // val ^ (-1 ^ (val ror shift)) ==> eon
12100 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
12101                          iRegL src1, iRegL src2,
12102                          immI src3, immL_M1 src4) %{
12103   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
12104   ins_cost(1.9 * INSN_COST);
12105   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
12106 
12107   ins_encode %{
12108     __ eon(as_Register($dst$$reg),
12109               as_Register($src1$$reg),
12110               as_Register($src2$$reg),
12111               Assembler::ROR,
12112               $src3$$constant & 0x3f);
12113   %}
12114 
12115   ins_pipe(ialu_reg_reg_shift);
12116 %}
12117 
12118 // This pattern is automatically generated from aarch64_ad.m4
12119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12120 // val ^ (-1 ^ (val << shift)) ==> eonw
12121 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
12122                          iRegIorL2I src1, iRegIorL2I src2,
12123                          immI src3, immI_M1 src4) %{
12124   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
12125   ins_cost(1.9 * INSN_COST);
12126   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
12127 
12128   ins_encode %{
12129     __ eonw(as_Register($dst$$reg),
12130               as_Register($src1$$reg),
12131               as_Register($src2$$reg),
12132               Assembler::LSL,
12133               $src3$$constant & 0x1f);
12134   %}
12135 
12136   ins_pipe(ialu_reg_reg_shift);
12137 %}
12138 
12139 // This pattern is automatically generated from aarch64_ad.m4
12140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12141 // val ^ (-1 ^ (val << shift)) ==> eon
12142 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
12143                          iRegL src1, iRegL src2,
12144                          immI src3, immL_M1 src4) %{
12145   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12146   ins_cost(1.9 * INSN_COST);
12147   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
12148 
12149   ins_encode %{
12150     __ eon(as_Register($dst$$reg),
12151               as_Register($src1$$reg),
12152               as_Register($src2$$reg),
12153               Assembler::LSL,
12154               $src3$$constant & 0x3f);
12155   %}
12156 
12157   ins_pipe(ialu_reg_reg_shift);
12158 %}
12159 
12160 // This pattern is automatically generated from aarch64_ad.m4
12161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12162 // val | (-1 ^ (val >>> shift)) ==> ornw
12163 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12164                          iRegIorL2I src1, iRegIorL2I src2,
12165                          immI src3, immI_M1 src4) %{
12166   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12167   ins_cost(1.9 * INSN_COST);
12168   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
12169 
12170   ins_encode %{
12171     __ ornw(as_Register($dst$$reg),
12172               as_Register($src1$$reg),
12173               as_Register($src2$$reg),
12174               Assembler::LSR,
12175               $src3$$constant & 0x1f);
12176   %}
12177 
12178   ins_pipe(ialu_reg_reg_shift);
12179 %}
12180 
12181 // This pattern is automatically generated from aarch64_ad.m4
12182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12183 // val | (-1 ^ (val >>> shift)) ==> orn
12184 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12185                          iRegL src1, iRegL src2,
12186                          immI src3, immL_M1 src4) %{
12187   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12188   ins_cost(1.9 * INSN_COST);
12189   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
12190 
12191   ins_encode %{
12192     __ orn(as_Register($dst$$reg),
12193               as_Register($src1$$reg),
12194               as_Register($src2$$reg),
12195               Assembler::LSR,
12196               $src3$$constant & 0x3f);
12197   %}
12198 
12199   ins_pipe(ialu_reg_reg_shift);
12200 %}
12201 
12202 // This pattern is automatically generated from aarch64_ad.m4
12203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12204 // val | (-1 ^ (val >> shift)) ==> ornw
12205 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12206                          iRegIorL2I src1, iRegIorL2I src2,
12207                          immI src3, immI_M1 src4) %{
12208   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12209   ins_cost(1.9 * INSN_COST);
12210   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12211 
12212   ins_encode %{
12213     __ ornw(as_Register($dst$$reg),
12214               as_Register($src1$$reg),
12215               as_Register($src2$$reg),
12216               Assembler::ASR,
12217               $src3$$constant & 0x1f);
12218   %}
12219 
12220   ins_pipe(ialu_reg_reg_shift);
12221 %}
12222 
12223 // This pattern is automatically generated from aarch64_ad.m4
12224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12225 // val | (-1 ^ (val >> shift)) ==> orn
12226 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12227                          iRegL src1, iRegL src2,
12228                          immI src3, immL_M1 src4) %{
12229   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12230   ins_cost(1.9 * INSN_COST);
12231   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12232 
12233   ins_encode %{
12234     __ orn(as_Register($dst$$reg),
12235               as_Register($src1$$reg),
12236               as_Register($src2$$reg),
12237               Assembler::ASR,
12238               $src3$$constant & 0x3f);
12239   %}
12240 
12241   ins_pipe(ialu_reg_reg_shift);
12242 %}
12243 
12244 // This pattern is automatically generated from aarch64_ad.m4
12245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12246 // val | (-1 ^ (val ror shift)) ==> ornw
12247 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12248                          iRegIorL2I src1, iRegIorL2I src2,
12249                          immI src3, immI_M1 src4) %{
12250   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12251   ins_cost(1.9 * INSN_COST);
12252   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12253 
12254   ins_encode %{
12255     __ ornw(as_Register($dst$$reg),
12256               as_Register($src1$$reg),
12257               as_Register($src2$$reg),
12258               Assembler::ROR,
12259               $src3$$constant & 0x1f);
12260   %}
12261 
12262   ins_pipe(ialu_reg_reg_shift);
12263 %}
12264 
12265 // This pattern is automatically generated from aarch64_ad.m4
12266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12267 // val | (-1 ^ (val ror shift)) ==> orn
12268 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12269                          iRegL src1, iRegL src2,
12270                          immI src3, immL_M1 src4) %{
12271   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12272   ins_cost(1.9 * INSN_COST);
12273   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12274 
12275   ins_encode %{
12276     __ orn(as_Register($dst$$reg),
12277               as_Register($src1$$reg),
12278               as_Register($src2$$reg),
12279               Assembler::ROR,
12280               $src3$$constant & 0x3f);
12281   %}
12282 
12283   ins_pipe(ialu_reg_reg_shift);
12284 %}
12285 
12286 // This pattern is automatically generated from aarch64_ad.m4
12287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12288 // val | (-1 ^ (val << shift)) ==> ornw
12289 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12290                          iRegIorL2I src1, iRegIorL2I src2,
12291                          immI src3, immI_M1 src4) %{
12292   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12293   ins_cost(1.9 * INSN_COST);
12294   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12295 
12296   ins_encode %{
12297     __ ornw(as_Register($dst$$reg),
12298               as_Register($src1$$reg),
12299               as_Register($src2$$reg),
12300               Assembler::LSL,
12301               $src3$$constant & 0x1f);
12302   %}
12303 
12304   ins_pipe(ialu_reg_reg_shift);
12305 %}
12306 
12307 // This pattern is automatically generated from aarch64_ad.m4
12308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12309 // val | (-1 ^ (val << shift)) ==> orn
12310 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12311                          iRegL src1, iRegL src2,
12312                          immI src3, immL_M1 src4) %{
12313   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12314   ins_cost(1.9 * INSN_COST);
12315   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12316 
12317   ins_encode %{
12318     __ orn(as_Register($dst$$reg),
12319               as_Register($src1$$reg),
12320               as_Register($src2$$reg),
12321               Assembler::LSL,
12322               $src3$$constant & 0x3f);
12323   %}
12324 
12325   ins_pipe(ialu_reg_reg_shift);
12326 %}
12327 
12328 // This pattern is automatically generated from aarch64_ad.m4
12329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12330 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12331                          iRegIorL2I src1, iRegIorL2I src2,
12332                          immI src3) %{
12333   match(Set dst (AndI src1 (URShiftI src2 src3)));
12334 
12335   ins_cost(1.9 * INSN_COST);
12336   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12337 
12338   ins_encode %{
12339     __ andw(as_Register($dst$$reg),
12340               as_Register($src1$$reg),
12341               as_Register($src2$$reg),
12342               Assembler::LSR,
12343               $src3$$constant & 0x1f);
12344   %}
12345 
12346   ins_pipe(ialu_reg_reg_shift);
12347 %}
12348 
12349 // This pattern is automatically generated from aarch64_ad.m4
12350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12351 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12352                          iRegL src1, iRegL src2,
12353                          immI src3) %{
12354   match(Set dst (AndL src1 (URShiftL src2 src3)));
12355 
12356   ins_cost(1.9 * INSN_COST);
12357   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12358 
12359   ins_encode %{
12360     __ andr(as_Register($dst$$reg),
12361               as_Register($src1$$reg),
12362               as_Register($src2$$reg),
12363               Assembler::LSR,
12364               $src3$$constant & 0x3f);
12365   %}
12366 
12367   ins_pipe(ialu_reg_reg_shift);
12368 %}
12369 
12370 // This pattern is automatically generated from aarch64_ad.m4
12371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12372 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12373                          iRegIorL2I src1, iRegIorL2I src2,
12374                          immI src3) %{
12375   match(Set dst (AndI src1 (RShiftI src2 src3)));
12376 
12377   ins_cost(1.9 * INSN_COST);
12378   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12379 
12380   ins_encode %{
12381     __ andw(as_Register($dst$$reg),
12382               as_Register($src1$$reg),
12383               as_Register($src2$$reg),
12384               Assembler::ASR,
12385               $src3$$constant & 0x1f);
12386   %}
12387 
12388   ins_pipe(ialu_reg_reg_shift);
12389 %}
12390 
12391 // This pattern is automatically generated from aarch64_ad.m4
12392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12393 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12394                          iRegL src1, iRegL src2,
12395                          immI src3) %{
12396   match(Set dst (AndL src1 (RShiftL src2 src3)));
12397 
12398   ins_cost(1.9 * INSN_COST);
12399   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12400 
12401   ins_encode %{
12402     __ andr(as_Register($dst$$reg),
12403               as_Register($src1$$reg),
12404               as_Register($src2$$reg),
12405               Assembler::ASR,
12406               $src3$$constant & 0x3f);
12407   %}
12408 
12409   ins_pipe(ialu_reg_reg_shift);
12410 %}
12411 
12412 // This pattern is automatically generated from aarch64_ad.m4
12413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12414 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12415                          iRegIorL2I src1, iRegIorL2I src2,
12416                          immI src3) %{
12417   match(Set dst (AndI src1 (LShiftI src2 src3)));
12418 
12419   ins_cost(1.9 * INSN_COST);
12420   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12421 
12422   ins_encode %{
12423     __ andw(as_Register($dst$$reg),
12424               as_Register($src1$$reg),
12425               as_Register($src2$$reg),
12426               Assembler::LSL,
12427               $src3$$constant & 0x1f);
12428   %}
12429 
12430   ins_pipe(ialu_reg_reg_shift);
12431 %}
12432 
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12436                          iRegL src1, iRegL src2,
12437                          immI src3) %{
12438   match(Set dst (AndL src1 (LShiftL src2 src3)));
12439 
12440   ins_cost(1.9 * INSN_COST);
12441   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12442 
12443   ins_encode %{
12444     __ andr(as_Register($dst$$reg),
12445               as_Register($src1$$reg),
12446               as_Register($src2$$reg),
12447               Assembler::LSL,
12448               $src3$$constant & 0x3f);
12449   %}
12450 
12451   ins_pipe(ialu_reg_reg_shift);
12452 %}
12453 
12454 // This pattern is automatically generated from aarch64_ad.m4
12455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12456 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12457                          iRegIorL2I src1, iRegIorL2I src2,
12458                          immI src3) %{
12459   match(Set dst (AndI src1 (RotateRight src2 src3)));
12460 
12461   ins_cost(1.9 * INSN_COST);
12462   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12463 
12464   ins_encode %{
12465     __ andw(as_Register($dst$$reg),
12466               as_Register($src1$$reg),
12467               as_Register($src2$$reg),
12468               Assembler::ROR,
12469               $src3$$constant & 0x1f);
12470   %}
12471 
12472   ins_pipe(ialu_reg_reg_shift);
12473 %}
12474 
12475 // This pattern is automatically generated from aarch64_ad.m4
12476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12477 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12478                          iRegL src1, iRegL src2,
12479                          immI src3) %{
12480   match(Set dst (AndL src1 (RotateRight src2 src3)));
12481 
12482   ins_cost(1.9 * INSN_COST);
12483   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12484 
12485   ins_encode %{
12486     __ andr(as_Register($dst$$reg),
12487               as_Register($src1$$reg),
12488               as_Register($src2$$reg),
12489               Assembler::ROR,
12490               $src3$$constant & 0x3f);
12491   %}
12492 
12493   ins_pipe(ialu_reg_reg_shift);
12494 %}
12495 
12496 // This pattern is automatically generated from aarch64_ad.m4
12497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12498 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12499                          iRegIorL2I src1, iRegIorL2I src2,
12500                          immI src3) %{
12501   match(Set dst (XorI src1 (URShiftI src2 src3)));
12502 
12503   ins_cost(1.9 * INSN_COST);
12504   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12505 
12506   ins_encode %{
12507     __ eorw(as_Register($dst$$reg),
12508               as_Register($src1$$reg),
12509               as_Register($src2$$reg),
12510               Assembler::LSR,
12511               $src3$$constant & 0x1f);
12512   %}
12513 
12514   ins_pipe(ialu_reg_reg_shift);
12515 %}
12516 
12517 // This pattern is automatically generated from aarch64_ad.m4
12518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12519 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12520                          iRegL src1, iRegL src2,
12521                          immI src3) %{
12522   match(Set dst (XorL src1 (URShiftL src2 src3)));
12523 
12524   ins_cost(1.9 * INSN_COST);
12525   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12526 
12527   ins_encode %{
12528     __ eor(as_Register($dst$$reg),
12529               as_Register($src1$$reg),
12530               as_Register($src2$$reg),
12531               Assembler::LSR,
12532               $src3$$constant & 0x3f);
12533   %}
12534 
12535   ins_pipe(ialu_reg_reg_shift);
12536 %}
12537 
12538 // This pattern is automatically generated from aarch64_ad.m4
12539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12540 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12541                          iRegIorL2I src1, iRegIorL2I src2,
12542                          immI src3) %{
12543   match(Set dst (XorI src1 (RShiftI src2 src3)));
12544 
12545   ins_cost(1.9 * INSN_COST);
12546   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12547 
12548   ins_encode %{
12549     __ eorw(as_Register($dst$$reg),
12550               as_Register($src1$$reg),
12551               as_Register($src2$$reg),
12552               Assembler::ASR,
12553               $src3$$constant & 0x1f);
12554   %}
12555 
12556   ins_pipe(ialu_reg_reg_shift);
12557 %}
12558 
12559 // This pattern is automatically generated from aarch64_ad.m4
12560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12561 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12562                          iRegL src1, iRegL src2,
12563                          immI src3) %{
12564   match(Set dst (XorL src1 (RShiftL src2 src3)));
12565 
12566   ins_cost(1.9 * INSN_COST);
12567   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12568 
12569   ins_encode %{
12570     __ eor(as_Register($dst$$reg),
12571               as_Register($src1$$reg),
12572               as_Register($src2$$reg),
12573               Assembler::ASR,
12574               $src3$$constant & 0x3f);
12575   %}
12576 
12577   ins_pipe(ialu_reg_reg_shift);
12578 %}
12579 
12580 // This pattern is automatically generated from aarch64_ad.m4
12581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12582 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12583                          iRegIorL2I src1, iRegIorL2I src2,
12584                          immI src3) %{
12585   match(Set dst (XorI src1 (LShiftI src2 src3)));
12586 
12587   ins_cost(1.9 * INSN_COST);
12588   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12589 
12590   ins_encode %{
12591     __ eorw(as_Register($dst$$reg),
12592               as_Register($src1$$reg),
12593               as_Register($src2$$reg),
12594               Assembler::LSL,
12595               $src3$$constant & 0x1f);
12596   %}
12597 
12598   ins_pipe(ialu_reg_reg_shift);
12599 %}
12600 
12601 // This pattern is automatically generated from aarch64_ad.m4
12602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12603 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12604                          iRegL src1, iRegL src2,
12605                          immI src3) %{
12606   match(Set dst (XorL src1 (LShiftL src2 src3)));
12607 
12608   ins_cost(1.9 * INSN_COST);
12609   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12610 
12611   ins_encode %{
12612     __ eor(as_Register($dst$$reg),
12613               as_Register($src1$$reg),
12614               as_Register($src2$$reg),
12615               Assembler::LSL,
12616               $src3$$constant & 0x3f);
12617   %}
12618 
12619   ins_pipe(ialu_reg_reg_shift);
12620 %}
12621 
12622 // This pattern is automatically generated from aarch64_ad.m4
12623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12624 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12625                          iRegIorL2I src1, iRegIorL2I src2,
12626                          immI src3) %{
12627   match(Set dst (XorI src1 (RotateRight src2 src3)));
12628 
12629   ins_cost(1.9 * INSN_COST);
12630   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12631 
12632   ins_encode %{
12633     __ eorw(as_Register($dst$$reg),
12634               as_Register($src1$$reg),
12635               as_Register($src2$$reg),
12636               Assembler::ROR,
12637               $src3$$constant & 0x1f);
12638   %}
12639 
12640   ins_pipe(ialu_reg_reg_shift);
12641 %}
12642 
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12646                          iRegL src1, iRegL src2,
12647                          immI src3) %{
12648   match(Set dst (XorL src1 (RotateRight src2 src3)));
12649 
12650   ins_cost(1.9 * INSN_COST);
12651   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12652 
12653   ins_encode %{
12654     __ eor(as_Register($dst$$reg),
12655               as_Register($src1$$reg),
12656               as_Register($src2$$reg),
12657               Assembler::ROR,
12658               $src3$$constant & 0x3f);
12659   %}
12660 
12661   ins_pipe(ialu_reg_reg_shift);
12662 %}
12663 
12664 // This pattern is automatically generated from aarch64_ad.m4
12665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12666 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12667                          iRegIorL2I src1, iRegIorL2I src2,
12668                          immI src3) %{
12669   match(Set dst (OrI src1 (URShiftI src2 src3)));
12670 
12671   ins_cost(1.9 * INSN_COST);
12672   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12673 
12674   ins_encode %{
12675     __ orrw(as_Register($dst$$reg),
12676               as_Register($src1$$reg),
12677               as_Register($src2$$reg),
12678               Assembler::LSR,
12679               $src3$$constant & 0x1f);
12680   %}
12681 
12682   ins_pipe(ialu_reg_reg_shift);
12683 %}
12684 
12685 // This pattern is automatically generated from aarch64_ad.m4
12686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12687 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12688                          iRegL src1, iRegL src2,
12689                          immI src3) %{
12690   match(Set dst (OrL src1 (URShiftL src2 src3)));
12691 
12692   ins_cost(1.9 * INSN_COST);
12693   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12694 
12695   ins_encode %{
12696     __ orr(as_Register($dst$$reg),
12697               as_Register($src1$$reg),
12698               as_Register($src2$$reg),
12699               Assembler::LSR,
12700               $src3$$constant & 0x3f);
12701   %}
12702 
12703   ins_pipe(ialu_reg_reg_shift);
12704 %}
12705 
12706 // This pattern is automatically generated from aarch64_ad.m4
12707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12708 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12709                          iRegIorL2I src1, iRegIorL2I src2,
12710                          immI src3) %{
12711   match(Set dst (OrI src1 (RShiftI src2 src3)));
12712 
12713   ins_cost(1.9 * INSN_COST);
12714   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12715 
12716   ins_encode %{
12717     __ orrw(as_Register($dst$$reg),
12718               as_Register($src1$$reg),
12719               as_Register($src2$$reg),
12720               Assembler::ASR,
12721               $src3$$constant & 0x1f);
12722   %}
12723 
12724   ins_pipe(ialu_reg_reg_shift);
12725 %}
12726 
12727 // This pattern is automatically generated from aarch64_ad.m4
12728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12729 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12730                          iRegL src1, iRegL src2,
12731                          immI src3) %{
12732   match(Set dst (OrL src1 (RShiftL src2 src3)));
12733 
12734   ins_cost(1.9 * INSN_COST);
12735   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12736 
12737   ins_encode %{
12738     __ orr(as_Register($dst$$reg),
12739               as_Register($src1$$reg),
12740               as_Register($src2$$reg),
12741               Assembler::ASR,
12742               $src3$$constant & 0x3f);
12743   %}
12744 
12745   ins_pipe(ialu_reg_reg_shift);
12746 %}
12747 
12748 // This pattern is automatically generated from aarch64_ad.m4
12749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12750 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12751                          iRegIorL2I src1, iRegIorL2I src2,
12752                          immI src3) %{
12753   match(Set dst (OrI src1 (LShiftI src2 src3)));
12754 
12755   ins_cost(1.9 * INSN_COST);
12756   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12757 
12758   ins_encode %{
12759     __ orrw(as_Register($dst$$reg),
12760               as_Register($src1$$reg),
12761               as_Register($src2$$reg),
12762               Assembler::LSL,
12763               $src3$$constant & 0x1f);
12764   %}
12765 
12766   ins_pipe(ialu_reg_reg_shift);
12767 %}
12768 
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12772                          iRegL src1, iRegL src2,
12773                          immI src3) %{
12774   match(Set dst (OrL src1 (LShiftL src2 src3)));
12775 
12776   ins_cost(1.9 * INSN_COST);
12777   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12778 
12779   ins_encode %{
12780     __ orr(as_Register($dst$$reg),
12781               as_Register($src1$$reg),
12782               as_Register($src2$$reg),
12783               Assembler::LSL,
12784               $src3$$constant & 0x3f);
12785   %}
12786 
12787   ins_pipe(ialu_reg_reg_shift);
12788 %}
12789 
12790 // This pattern is automatically generated from aarch64_ad.m4
12791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12792 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12793                          iRegIorL2I src1, iRegIorL2I src2,
12794                          immI src3) %{
12795   match(Set dst (OrI src1 (RotateRight src2 src3)));
12796 
12797   ins_cost(1.9 * INSN_COST);
12798   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12799 
12800   ins_encode %{
12801     __ orrw(as_Register($dst$$reg),
12802               as_Register($src1$$reg),
12803               as_Register($src2$$reg),
12804               Assembler::ROR,
12805               $src3$$constant & 0x1f);
12806   %}
12807 
12808   ins_pipe(ialu_reg_reg_shift);
12809 %}
12810 
12811 // This pattern is automatically generated from aarch64_ad.m4
12812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12813 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12814                          iRegL src1, iRegL src2,
12815                          immI src3) %{
12816   match(Set dst (OrL src1 (RotateRight src2 src3)));
12817 
12818   ins_cost(1.9 * INSN_COST);
12819   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12820 
12821   ins_encode %{
12822     __ orr(as_Register($dst$$reg),
12823               as_Register($src1$$reg),
12824               as_Register($src2$$reg),
12825               Assembler::ROR,
12826               $src3$$constant & 0x3f);
12827   %}
12828 
12829   ins_pipe(ialu_reg_reg_shift);
12830 %}
12831 
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12835                          iRegIorL2I src1, iRegIorL2I src2,
12836                          immI src3) %{
12837   match(Set dst (AddI src1 (URShiftI src2 src3)));
12838 
12839   ins_cost(1.9 * INSN_COST);
12840   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12841 
12842   ins_encode %{
12843     __ addw(as_Register($dst$$reg),
12844               as_Register($src1$$reg),
12845               as_Register($src2$$reg),
12846               Assembler::LSR,
12847               $src3$$constant & 0x1f);
12848   %}
12849 
12850   ins_pipe(ialu_reg_reg_shift);
12851 %}
12852 
12853 // This pattern is automatically generated from aarch64_ad.m4
12854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12855 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12856                          iRegL src1, iRegL src2,
12857                          immI src3) %{
12858   match(Set dst (AddL src1 (URShiftL src2 src3)));
12859 
12860   ins_cost(1.9 * INSN_COST);
12861   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12862 
12863   ins_encode %{
12864     __ add(as_Register($dst$$reg),
12865               as_Register($src1$$reg),
12866               as_Register($src2$$reg),
12867               Assembler::LSR,
12868               $src3$$constant & 0x3f);
12869   %}
12870 
12871   ins_pipe(ialu_reg_reg_shift);
12872 %}
12873 
12874 // This pattern is automatically generated from aarch64_ad.m4
12875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12876 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12877                          iRegIorL2I src1, iRegIorL2I src2,
12878                          immI src3) %{
12879   match(Set dst (AddI src1 (RShiftI src2 src3)));
12880 
12881   ins_cost(1.9 * INSN_COST);
12882   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12883 
12884   ins_encode %{
12885     __ addw(as_Register($dst$$reg),
12886               as_Register($src1$$reg),
12887               as_Register($src2$$reg),
12888               Assembler::ASR,
12889               $src3$$constant & 0x1f);
12890   %}
12891 
12892   ins_pipe(ialu_reg_reg_shift);
12893 %}
12894 
12895 // This pattern is automatically generated from aarch64_ad.m4
12896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12897 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12898                          iRegL src1, iRegL src2,
12899                          immI src3) %{
12900   match(Set dst (AddL src1 (RShiftL src2 src3)));
12901 
12902   ins_cost(1.9 * INSN_COST);
12903   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12904 
12905   ins_encode %{
12906     __ add(as_Register($dst$$reg),
12907               as_Register($src1$$reg),
12908               as_Register($src2$$reg),
12909               Assembler::ASR,
12910               $src3$$constant & 0x3f);
12911   %}
12912 
12913   ins_pipe(ialu_reg_reg_shift);
12914 %}
12915 
12916 // This pattern is automatically generated from aarch64_ad.m4
12917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12918 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12919                          iRegIorL2I src1, iRegIorL2I src2,
12920                          immI src3) %{
12921   match(Set dst (AddI src1 (LShiftI src2 src3)));
12922 
12923   ins_cost(1.9 * INSN_COST);
12924   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12925 
12926   ins_encode %{
12927     __ addw(as_Register($dst$$reg),
12928               as_Register($src1$$reg),
12929               as_Register($src2$$reg),
12930               Assembler::LSL,
12931               $src3$$constant & 0x1f);
12932   %}
12933 
12934   ins_pipe(ialu_reg_reg_shift);
12935 %}
12936 
12937 // This pattern is automatically generated from aarch64_ad.m4
12938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12939 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12940                          iRegL src1, iRegL src2,
12941                          immI src3) %{
12942   match(Set dst (AddL src1 (LShiftL src2 src3)));
12943 
12944   ins_cost(1.9 * INSN_COST);
12945   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12946 
12947   ins_encode %{
12948     __ add(as_Register($dst$$reg),
12949               as_Register($src1$$reg),
12950               as_Register($src2$$reg),
12951               Assembler::LSL,
12952               $src3$$constant & 0x3f);
12953   %}
12954 
12955   ins_pipe(ialu_reg_reg_shift);
12956 %}
12957 
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12961                          iRegIorL2I src1, iRegIorL2I src2,
12962                          immI src3) %{
12963   match(Set dst (SubI src1 (URShiftI src2 src3)));
12964 
12965   ins_cost(1.9 * INSN_COST);
12966   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12967 
12968   ins_encode %{
12969     __ subw(as_Register($dst$$reg),
12970               as_Register($src1$$reg),
12971               as_Register($src2$$reg),
12972               Assembler::LSR,
12973               $src3$$constant & 0x1f);
12974   %}
12975 
12976   ins_pipe(ialu_reg_reg_shift);
12977 %}
12978 
12979 // This pattern is automatically generated from aarch64_ad.m4
12980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12981 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12982                          iRegL src1, iRegL src2,
12983                          immI src3) %{
12984   match(Set dst (SubL src1 (URShiftL src2 src3)));
12985 
12986   ins_cost(1.9 * INSN_COST);
12987   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12988 
12989   ins_encode %{
12990     __ sub(as_Register($dst$$reg),
12991               as_Register($src1$$reg),
12992               as_Register($src2$$reg),
12993               Assembler::LSR,
12994               $src3$$constant & 0x3f);
12995   %}
12996 
12997   ins_pipe(ialu_reg_reg_shift);
12998 %}
12999 
13000 // This pattern is automatically generated from aarch64_ad.m4
13001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13002 instruct SubI_reg_RShift_reg(iRegINoSp dst,
13003                          iRegIorL2I src1, iRegIorL2I src2,
13004                          immI src3) %{
13005   match(Set dst (SubI src1 (RShiftI src2 src3)));
13006 
13007   ins_cost(1.9 * INSN_COST);
13008   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
13009 
13010   ins_encode %{
13011     __ subw(as_Register($dst$$reg),
13012               as_Register($src1$$reg),
13013               as_Register($src2$$reg),
13014               Assembler::ASR,
13015               $src3$$constant & 0x1f);
13016   %}
13017 
13018   ins_pipe(ialu_reg_reg_shift);
13019 %}
13020 
13021 // This pattern is automatically generated from aarch64_ad.m4
13022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13023 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
13024                          iRegL src1, iRegL src2,
13025                          immI src3) %{
13026   match(Set dst (SubL src1 (RShiftL src2 src3)));
13027 
13028   ins_cost(1.9 * INSN_COST);
13029   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
13030 
13031   ins_encode %{
13032     __ sub(as_Register($dst$$reg),
13033               as_Register($src1$$reg),
13034               as_Register($src2$$reg),
13035               Assembler::ASR,
13036               $src3$$constant & 0x3f);
13037   %}
13038 
13039   ins_pipe(ialu_reg_reg_shift);
13040 %}
13041 
13042 // This pattern is automatically generated from aarch64_ad.m4
13043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13044 instruct SubI_reg_LShift_reg(iRegINoSp dst,
13045                          iRegIorL2I src1, iRegIorL2I src2,
13046                          immI src3) %{
13047   match(Set dst (SubI src1 (LShiftI src2 src3)));
13048 
13049   ins_cost(1.9 * INSN_COST);
13050   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
13051 
13052   ins_encode %{
13053     __ subw(as_Register($dst$$reg),
13054               as_Register($src1$$reg),
13055               as_Register($src2$$reg),
13056               Assembler::LSL,
13057               $src3$$constant & 0x1f);
13058   %}
13059 
13060   ins_pipe(ialu_reg_reg_shift);
13061 %}
13062 
13063 // This pattern is automatically generated from aarch64_ad.m4
13064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13065 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
13066                          iRegL src1, iRegL src2,
13067                          immI src3) %{
13068   match(Set dst (SubL src1 (LShiftL src2 src3)));
13069 
13070   ins_cost(1.9 * INSN_COST);
13071   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
13072 
13073   ins_encode %{
13074     __ sub(as_Register($dst$$reg),
13075               as_Register($src1$$reg),
13076               as_Register($src2$$reg),
13077               Assembler::LSL,
13078               $src3$$constant & 0x3f);
13079   %}
13080 
13081   ins_pipe(ialu_reg_reg_shift);
13082 %}
13083 
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086 
13087 // Shift Left followed by Shift Right.
13088 // This idiom is used by the compiler for the i2b bytecode etc.
13089 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13090 %{
13091   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
13092   ins_cost(INSN_COST * 2);
13093   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13094   ins_encode %{
13095     int lshift = $lshift_count$$constant & 63;
13096     int rshift = $rshift_count$$constant & 63;
13097     int s = 63 - lshift;
13098     int r = (rshift - lshift) & 63;
13099     __ sbfm(as_Register($dst$$reg),
13100             as_Register($src$$reg),
13101             r, s);
13102   %}
13103 
13104   ins_pipe(ialu_reg_shift);
13105 %}
13106 
13107 // This pattern is automatically generated from aarch64_ad.m4
13108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13109 
13110 // Shift Left followed by Shift Right.
13111 // This idiom is used by the compiler for the i2b bytecode etc.
13112 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13113 %{
13114   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
13115   ins_cost(INSN_COST * 2);
13116   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13117   ins_encode %{
13118     int lshift = $lshift_count$$constant & 31;
13119     int rshift = $rshift_count$$constant & 31;
13120     int s = 31 - lshift;
13121     int r = (rshift - lshift) & 31;
13122     __ sbfmw(as_Register($dst$$reg),
13123             as_Register($src$$reg),
13124             r, s);
13125   %}
13126 
13127   ins_pipe(ialu_reg_shift);
13128 %}
13129 
13130 // This pattern is automatically generated from aarch64_ad.m4
13131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13132 
13133 // Shift Left followed by Shift Right.
13134 // This idiom is used by the compiler for the i2b bytecode etc.
13135 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13136 %{
13137   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
13138   ins_cost(INSN_COST * 2);
13139   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13140   ins_encode %{
13141     int lshift = $lshift_count$$constant & 63;
13142     int rshift = $rshift_count$$constant & 63;
13143     int s = 63 - lshift;
13144     int r = (rshift - lshift) & 63;
13145     __ ubfm(as_Register($dst$$reg),
13146             as_Register($src$$reg),
13147             r, s);
13148   %}
13149 
13150   ins_pipe(ialu_reg_shift);
13151 %}
13152 
13153 // This pattern is automatically generated from aarch64_ad.m4
13154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13155 
13156 // Shift Left followed by Shift Right.
13157 // This idiom is used by the compiler for the i2b bytecode etc.
13158 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13159 %{
13160   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13161   ins_cost(INSN_COST * 2);
13162   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13163   ins_encode %{
13164     int lshift = $lshift_count$$constant & 31;
13165     int rshift = $rshift_count$$constant & 31;
13166     int s = 31 - lshift;
13167     int r = (rshift - lshift) & 31;
13168     __ ubfmw(as_Register($dst$$reg),
13169             as_Register($src$$reg),
13170             r, s);
13171   %}
13172 
13173   ins_pipe(ialu_reg_shift);
13174 %}
13175 
13176 // Bitfield extract with shift & mask
13177 
13178 // This pattern is automatically generated from aarch64_ad.m4
13179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13180 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13181 %{
13182   match(Set dst (AndI (URShiftI src rshift) mask));
13183   // Make sure we are not going to exceed what ubfxw can do.
13184   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13185 
13186   ins_cost(INSN_COST);
13187   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13188   ins_encode %{
13189     int rshift = $rshift$$constant & 31;
13190     intptr_t mask = $mask$$constant;
13191     int width = exact_log2(mask+1);
13192     __ ubfxw(as_Register($dst$$reg),
13193             as_Register($src$$reg), rshift, width);
13194   %}
13195   ins_pipe(ialu_reg_shift);
13196 %}
13197 
13198 // This pattern is automatically generated from aarch64_ad.m4
13199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13200 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13201 %{
13202   match(Set dst (AndL (URShiftL src rshift) mask));
13203   // Make sure we are not going to exceed what ubfx can do.
13204   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13205 
13206   ins_cost(INSN_COST);
13207   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13208   ins_encode %{
13209     int rshift = $rshift$$constant & 63;
13210     intptr_t mask = $mask$$constant;
13211     int width = exact_log2_long(mask+1);
13212     __ ubfx(as_Register($dst$$reg),
13213             as_Register($src$$reg), rshift, width);
13214   %}
13215   ins_pipe(ialu_reg_shift);
13216 %}
13217 
13218 
13219 // This pattern is automatically generated from aarch64_ad.m4
13220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13221 
13222 // We can use ubfx when extending an And with a mask when we know mask
13223 // is positive.  We know that because immI_bitmask guarantees it.
13224 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13225 %{
13226   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13227   // Make sure we are not going to exceed what ubfxw can do.
13228   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13229 
13230   ins_cost(INSN_COST * 2);
13231   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13232   ins_encode %{
13233     int rshift = $rshift$$constant & 31;
13234     intptr_t mask = $mask$$constant;
13235     int width = exact_log2(mask+1);
13236     __ ubfx(as_Register($dst$$reg),
13237             as_Register($src$$reg), rshift, width);
13238   %}
13239   ins_pipe(ialu_reg_shift);
13240 %}
13241 
13242 
13243 // This pattern is automatically generated from aarch64_ad.m4
13244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13245 
13246 // We can use ubfiz when masking by a positive number and then left shifting the result.
13247 // We know that the mask is positive because immI_bitmask guarantees it.
13248 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13249 %{
13250   match(Set dst (LShiftI (AndI src mask) lshift));
13251   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13252 
13253   ins_cost(INSN_COST);
13254   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13255   ins_encode %{
13256     int lshift = $lshift$$constant & 31;
13257     intptr_t mask = $mask$$constant;
13258     int width = exact_log2(mask+1);
13259     __ ubfizw(as_Register($dst$$reg),
13260           as_Register($src$$reg), lshift, width);
13261   %}
13262   ins_pipe(ialu_reg_shift);
13263 %}
13264 
13265 // This pattern is automatically generated from aarch64_ad.m4
13266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13267 
13268 // We can use ubfiz when masking by a positive number and then left shifting the result.
13269 // We know that the mask is positive because immL_bitmask guarantees it.
13270 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13271 %{
13272   match(Set dst (LShiftL (AndL src mask) lshift));
13273   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13274 
13275   ins_cost(INSN_COST);
13276   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13277   ins_encode %{
13278     int lshift = $lshift$$constant & 63;
13279     intptr_t mask = $mask$$constant;
13280     int width = exact_log2_long(mask+1);
13281     __ ubfiz(as_Register($dst$$reg),
13282           as_Register($src$$reg), lshift, width);
13283   %}
13284   ins_pipe(ialu_reg_shift);
13285 %}
13286 
13287 // This pattern is automatically generated from aarch64_ad.m4
13288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13289 
13290 // We can use ubfiz when masking by a positive number and then left shifting the result.
13291 // We know that the mask is positive because immI_bitmask guarantees it.
13292 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13293 %{
13294   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13295   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13296 
13297   ins_cost(INSN_COST);
13298   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13299   ins_encode %{
13300     int lshift = $lshift$$constant & 31;
13301     intptr_t mask = $mask$$constant;
13302     int width = exact_log2(mask+1);
13303     __ ubfizw(as_Register($dst$$reg),
13304           as_Register($src$$reg), lshift, width);
13305   %}
13306   ins_pipe(ialu_reg_shift);
13307 %}
13308 
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 
13312 // We can use ubfiz when masking by a positive number and then left shifting the result.
13313 // We know that the mask is positive because immL_bitmask guarantees it.
13314 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13315 %{
13316   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13317   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13318 
13319   ins_cost(INSN_COST);
13320   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13321   ins_encode %{
13322     int lshift = $lshift$$constant & 63;
13323     intptr_t mask = $mask$$constant;
13324     int width = exact_log2_long(mask+1);
13325     __ ubfiz(as_Register($dst$$reg),
13326           as_Register($src$$reg), lshift, width);
13327   %}
13328   ins_pipe(ialu_reg_shift);
13329 %}
13330 
13331 
13332 // This pattern is automatically generated from aarch64_ad.m4
13333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13334 
13335 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13336 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13337 %{
13338   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13339   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13340 
13341   ins_cost(INSN_COST);
13342   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13343   ins_encode %{
13344     int lshift = $lshift$$constant & 63;
13345     intptr_t mask = $mask$$constant;
13346     int width = exact_log2(mask+1);
13347     __ ubfiz(as_Register($dst$$reg),
13348              as_Register($src$$reg), lshift, width);
13349   %}
13350   ins_pipe(ialu_reg_shift);
13351 %}
13352 
13353 // This pattern is automatically generated from aarch64_ad.m4
13354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13355 
13356 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13357 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13358 %{
13359   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13360   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13361 
13362   ins_cost(INSN_COST);
13363   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13364   ins_encode %{
13365     int lshift = $lshift$$constant & 31;
13366     intptr_t mask = $mask$$constant;
13367     int width = exact_log2(mask+1);
13368     __ ubfiz(as_Register($dst$$reg),
13369              as_Register($src$$reg), lshift, width);
13370   %}
13371   ins_pipe(ialu_reg_shift);
13372 %}
13373 
13374 // This pattern is automatically generated from aarch64_ad.m4
13375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13376 
13377 // Can skip int2long conversions after AND with small bitmask
13378 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13379 %{
13380   match(Set dst (ConvI2L (AndI src msk)));
13381   ins_cost(INSN_COST);
13382   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13383   ins_encode %{
13384     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13385   %}
13386   ins_pipe(ialu_reg_shift);
13387 %}
13388 
13389 
13390 // Rotations
13391 // This pattern is automatically generated from aarch64_ad.m4
13392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13393 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13394 %{
13395   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13396   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13397 
13398   ins_cost(INSN_COST);
13399   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13400 
13401   ins_encode %{
13402     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13403             $rshift$$constant & 63);
13404   %}
13405   ins_pipe(ialu_reg_reg_extr);
13406 %}
13407 
13408 
13409 // This pattern is automatically generated from aarch64_ad.m4
13410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13411 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13412 %{
13413   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13414   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13415 
13416   ins_cost(INSN_COST);
13417   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13418 
13419   ins_encode %{
13420     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13421             $rshift$$constant & 31);
13422   %}
13423   ins_pipe(ialu_reg_reg_extr);
13424 %}
13425 
13426 
13427 // This pattern is automatically generated from aarch64_ad.m4
13428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13429 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13430 %{
13431   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13432   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13433 
13434   ins_cost(INSN_COST);
13435   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13436 
13437   ins_encode %{
13438     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13439             $rshift$$constant & 63);
13440   %}
13441   ins_pipe(ialu_reg_reg_extr);
13442 %}
13443 
13444 
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13448 %{
13449   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13450   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13451 
13452   ins_cost(INSN_COST);
13453   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13454 
13455   ins_encode %{
13456     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13457             $rshift$$constant & 31);
13458   %}
13459   ins_pipe(ialu_reg_reg_extr);
13460 %}
13461 
13462 
13463 // This pattern is automatically generated from aarch64_ad.m4
13464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13465 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13466 %{
13467   match(Set dst (RotateRight src shift));
13468 
13469   ins_cost(INSN_COST);
13470   format %{ "ror    $dst, $src, $shift" %}
13471 
13472   ins_encode %{
13473      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13474                $shift$$constant & 0x1f);
13475   %}
13476   ins_pipe(ialu_reg_reg_vshift);
13477 %}
13478 
13479 // This pattern is automatically generated from aarch64_ad.m4
13480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13481 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13482 %{
13483   match(Set dst (RotateRight src shift));
13484 
13485   ins_cost(INSN_COST);
13486   format %{ "ror    $dst, $src, $shift" %}
13487 
13488   ins_encode %{
13489      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13490                $shift$$constant & 0x3f);
13491   %}
13492   ins_pipe(ialu_reg_reg_vshift);
13493 %}
13494 
13495 // This pattern is automatically generated from aarch64_ad.m4
13496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13497 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13498 %{
13499   match(Set dst (RotateRight src shift));
13500 
13501   ins_cost(INSN_COST);
13502   format %{ "ror    $dst, $src, $shift" %}
13503 
13504   ins_encode %{
13505      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13506   %}
13507   ins_pipe(ialu_reg_reg_vshift);
13508 %}
13509 
13510 // This pattern is automatically generated from aarch64_ad.m4
13511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13512 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13513 %{
13514   match(Set dst (RotateRight src shift));
13515 
13516   ins_cost(INSN_COST);
13517   format %{ "ror    $dst, $src, $shift" %}
13518 
13519   ins_encode %{
13520      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13521   %}
13522   ins_pipe(ialu_reg_reg_vshift);
13523 %}
13524 
13525 // This pattern is automatically generated from aarch64_ad.m4
13526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13527 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13528 %{
13529   match(Set dst (RotateLeft src shift));
13530 
13531   ins_cost(INSN_COST);
13532   format %{ "rol    $dst, $src, $shift" %}
13533 
13534   ins_encode %{
13535      __ subw(rscratch1, zr, as_Register($shift$$reg));
13536      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13537   %}
13538   ins_pipe(ialu_reg_reg_vshift);
13539 %}
13540 
13541 // This pattern is automatically generated from aarch64_ad.m4
13542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13543 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13544 %{
13545   match(Set dst (RotateLeft src shift));
13546 
13547   ins_cost(INSN_COST);
13548   format %{ "rol    $dst, $src, $shift" %}
13549 
13550   ins_encode %{
13551      __ subw(rscratch1, zr, as_Register($shift$$reg));
13552      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13553   %}
13554   ins_pipe(ialu_reg_reg_vshift);
13555 %}
13556 
13557 
13558 // Add/subtract (extended)
13559 
13560 // This pattern is automatically generated from aarch64_ad.m4
13561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13562 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13563 %{
13564   match(Set dst (AddL src1 (ConvI2L src2)));
13565   ins_cost(INSN_COST);
13566   format %{ "add  $dst, $src1, $src2, sxtw" %}
13567 
13568    ins_encode %{
13569      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13570             as_Register($src2$$reg), ext::sxtw);
13571    %}
13572   ins_pipe(ialu_reg_reg);
13573 %}
13574 
13575 // This pattern is automatically generated from aarch64_ad.m4
13576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13577 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13578 %{
13579   match(Set dst (SubL src1 (ConvI2L src2)));
13580   ins_cost(INSN_COST);
13581   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13582 
13583    ins_encode %{
13584      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13585             as_Register($src2$$reg), ext::sxtw);
13586    %}
13587   ins_pipe(ialu_reg_reg);
13588 %}
13589 
13590 // This pattern is automatically generated from aarch64_ad.m4
13591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13592 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13593 %{
13594   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13595   ins_cost(INSN_COST);
13596   format %{ "add  $dst, $src1, $src2, sxth" %}
13597 
13598    ins_encode %{
13599      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13600             as_Register($src2$$reg), ext::sxth);
13601    %}
13602   ins_pipe(ialu_reg_reg);
13603 %}
13604 
13605 // This pattern is automatically generated from aarch64_ad.m4
13606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13607 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13608 %{
13609   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13610   ins_cost(INSN_COST);
13611   format %{ "add  $dst, $src1, $src2, sxtb" %}
13612 
13613    ins_encode %{
13614      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13615             as_Register($src2$$reg), ext::sxtb);
13616    %}
13617   ins_pipe(ialu_reg_reg);
13618 %}
13619 
13620 // This pattern is automatically generated from aarch64_ad.m4
13621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13622 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13623 %{
13624   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13625   ins_cost(INSN_COST);
13626   format %{ "add  $dst, $src1, $src2, uxtb" %}
13627 
13628    ins_encode %{
13629      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13630             as_Register($src2$$reg), ext::uxtb);
13631    %}
13632   ins_pipe(ialu_reg_reg);
13633 %}
13634 
13635 // This pattern is automatically generated from aarch64_ad.m4
13636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13637 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13638 %{
13639   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13640   ins_cost(INSN_COST);
13641   format %{ "add  $dst, $src1, $src2, sxth" %}
13642 
13643    ins_encode %{
13644      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13645             as_Register($src2$$reg), ext::sxth);
13646    %}
13647   ins_pipe(ialu_reg_reg);
13648 %}
13649 
13650 // This pattern is automatically generated from aarch64_ad.m4
13651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13652 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13653 %{
13654   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13655   ins_cost(INSN_COST);
13656   format %{ "add  $dst, $src1, $src2, sxtw" %}
13657 
13658    ins_encode %{
13659      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13660             as_Register($src2$$reg), ext::sxtw);
13661    %}
13662   ins_pipe(ialu_reg_reg);
13663 %}
13664 
13665 // This pattern is automatically generated from aarch64_ad.m4
13666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13667 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13668 %{
13669   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13670   ins_cost(INSN_COST);
13671   format %{ "add  $dst, $src1, $src2, sxtb" %}
13672 
13673    ins_encode %{
13674      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13675             as_Register($src2$$reg), ext::sxtb);
13676    %}
13677   ins_pipe(ialu_reg_reg);
13678 %}
13679 
13680 // This pattern is automatically generated from aarch64_ad.m4
13681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13682 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13683 %{
13684   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13685   ins_cost(INSN_COST);
13686   format %{ "add  $dst, $src1, $src2, uxtb" %}
13687 
13688    ins_encode %{
13689      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13690             as_Register($src2$$reg), ext::uxtb);
13691    %}
13692   ins_pipe(ialu_reg_reg);
13693 %}
13694 
13695 // This pattern is automatically generated from aarch64_ad.m4
13696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13697 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13698 %{
13699   match(Set dst (AddI src1 (AndI src2 mask)));
13700   ins_cost(INSN_COST);
13701   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13702 
13703    ins_encode %{
13704      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13705             as_Register($src2$$reg), ext::uxtb);
13706    %}
13707   ins_pipe(ialu_reg_reg);
13708 %}
13709 
13710 // This pattern is automatically generated from aarch64_ad.m4
13711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13712 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13713 %{
13714   match(Set dst (AddI src1 (AndI src2 mask)));
13715   ins_cost(INSN_COST);
13716   format %{ "addw  $dst, $src1, $src2, uxth" %}
13717 
13718    ins_encode %{
13719      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13720             as_Register($src2$$reg), ext::uxth);
13721    %}
13722   ins_pipe(ialu_reg_reg);
13723 %}
13724 
13725 // This pattern is automatically generated from aarch64_ad.m4
13726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13727 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13728 %{
13729   match(Set dst (AddL src1 (AndL src2 mask)));
13730   ins_cost(INSN_COST);
13731   format %{ "add  $dst, $src1, $src2, uxtb" %}
13732 
13733    ins_encode %{
13734      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13735             as_Register($src2$$reg), ext::uxtb);
13736    %}
13737   ins_pipe(ialu_reg_reg);
13738 %}
13739 
13740 // This pattern is automatically generated from aarch64_ad.m4
13741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13742 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13743 %{
13744   match(Set dst (AddL src1 (AndL src2 mask)));
13745   ins_cost(INSN_COST);
13746   format %{ "add  $dst, $src1, $src2, uxth" %}
13747 
13748    ins_encode %{
13749      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13750             as_Register($src2$$reg), ext::uxth);
13751    %}
13752   ins_pipe(ialu_reg_reg);
13753 %}
13754 
13755 // This pattern is automatically generated from aarch64_ad.m4
13756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13757 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13758 %{
13759   match(Set dst (AddL src1 (AndL src2 mask)));
13760   ins_cost(INSN_COST);
13761   format %{ "add  $dst, $src1, $src2, uxtw" %}
13762 
13763    ins_encode %{
13764      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13765             as_Register($src2$$reg), ext::uxtw);
13766    %}
13767   ins_pipe(ialu_reg_reg);
13768 %}
13769 
13770 // This pattern is automatically generated from aarch64_ad.m4
13771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13772 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13773 %{
13774   match(Set dst (SubI src1 (AndI src2 mask)));
13775   ins_cost(INSN_COST);
13776   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13777 
13778    ins_encode %{
13779      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13780             as_Register($src2$$reg), ext::uxtb);
13781    %}
13782   ins_pipe(ialu_reg_reg);
13783 %}
13784 
13785 // This pattern is automatically generated from aarch64_ad.m4
13786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13787 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13788 %{
13789   match(Set dst (SubI src1 (AndI src2 mask)));
13790   ins_cost(INSN_COST);
13791   format %{ "subw  $dst, $src1, $src2, uxth" %}
13792 
13793    ins_encode %{
13794      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13795             as_Register($src2$$reg), ext::uxth);
13796    %}
13797   ins_pipe(ialu_reg_reg);
13798 %}
13799 
13800 // This pattern is automatically generated from aarch64_ad.m4
13801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13802 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13803 %{
13804   match(Set dst (SubL src1 (AndL src2 mask)));
13805   ins_cost(INSN_COST);
13806   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13807 
13808    ins_encode %{
13809      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13810             as_Register($src2$$reg), ext::uxtb);
13811    %}
13812   ins_pipe(ialu_reg_reg);
13813 %}
13814 
13815 // This pattern is automatically generated from aarch64_ad.m4
13816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13817 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13818 %{
13819   match(Set dst (SubL src1 (AndL src2 mask)));
13820   ins_cost(INSN_COST);
13821   format %{ "sub  $dst, $src1, $src2, uxth" %}
13822 
13823    ins_encode %{
13824      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13825             as_Register($src2$$reg), ext::uxth);
13826    %}
13827   ins_pipe(ialu_reg_reg);
13828 %}
13829 
13830 // This pattern is automatically generated from aarch64_ad.m4
13831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13832 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13833 %{
13834   match(Set dst (SubL src1 (AndL src2 mask)));
13835   ins_cost(INSN_COST);
13836   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13837 
13838    ins_encode %{
13839      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13840             as_Register($src2$$reg), ext::uxtw);
13841    %}
13842   ins_pipe(ialu_reg_reg);
13843 %}
13844 
13845 
13846 // This pattern is automatically generated from aarch64_ad.m4
13847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13848 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13849 %{
13850   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13851   ins_cost(1.9 * INSN_COST);
13852   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13853 
13854    ins_encode %{
13855      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13856             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13857    %}
13858   ins_pipe(ialu_reg_reg_shift);
13859 %}
13860 
13861 // This pattern is automatically generated from aarch64_ad.m4
13862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13863 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13864 %{
13865   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13866   ins_cost(1.9 * INSN_COST);
13867   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13868 
13869    ins_encode %{
13870      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13871             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13872    %}
13873   ins_pipe(ialu_reg_reg_shift);
13874 %}
13875 
13876 // This pattern is automatically generated from aarch64_ad.m4
13877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13878 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13879 %{
13880   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13881   ins_cost(1.9 * INSN_COST);
13882   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13883 
13884    ins_encode %{
13885      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13886             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13887    %}
13888   ins_pipe(ialu_reg_reg_shift);
13889 %}
13890 
13891 // This pattern is automatically generated from aarch64_ad.m4
13892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13893 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13894 %{
13895   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13896   ins_cost(1.9 * INSN_COST);
13897   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13898 
13899    ins_encode %{
13900      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13901             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13902    %}
13903   ins_pipe(ialu_reg_reg_shift);
13904 %}
13905 
13906 // This pattern is automatically generated from aarch64_ad.m4
13907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13908 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13909 %{
13910   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13911   ins_cost(1.9 * INSN_COST);
13912   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13913 
13914    ins_encode %{
13915      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13916             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13917    %}
13918   ins_pipe(ialu_reg_reg_shift);
13919 %}
13920 
13921 // This pattern is automatically generated from aarch64_ad.m4
13922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13923 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13924 %{
13925   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13926   ins_cost(1.9 * INSN_COST);
13927   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13928 
13929    ins_encode %{
13930      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13931             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13932    %}
13933   ins_pipe(ialu_reg_reg_shift);
13934 %}
13935 
13936 // This pattern is automatically generated from aarch64_ad.m4
13937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13938 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13939 %{
13940   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13941   ins_cost(1.9 * INSN_COST);
13942   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13943 
13944    ins_encode %{
13945      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13946             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13947    %}
13948   ins_pipe(ialu_reg_reg_shift);
13949 %}
13950 
13951 // This pattern is automatically generated from aarch64_ad.m4
13952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13953 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13954 %{
13955   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13956   ins_cost(1.9 * INSN_COST);
13957   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13958 
13959    ins_encode %{
13960      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13961             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13962    %}
13963   ins_pipe(ialu_reg_reg_shift);
13964 %}
13965 
13966 // This pattern is automatically generated from aarch64_ad.m4
13967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13968 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13969 %{
13970   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13971   ins_cost(1.9 * INSN_COST);
13972   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13973 
13974    ins_encode %{
13975      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13976             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13977    %}
13978   ins_pipe(ialu_reg_reg_shift);
13979 %}
13980 
13981 // This pattern is automatically generated from aarch64_ad.m4
13982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13983 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13984 %{
13985   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13986   ins_cost(1.9 * INSN_COST);
13987   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13988 
13989    ins_encode %{
13990      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13991             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13992    %}
13993   ins_pipe(ialu_reg_reg_shift);
13994 %}
13995 
13996 // This pattern is automatically generated from aarch64_ad.m4
13997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13998 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13999 %{
14000   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
14001   ins_cost(1.9 * INSN_COST);
14002   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
14003 
14004    ins_encode %{
14005      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14006             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
14007    %}
14008   ins_pipe(ialu_reg_reg_shift);
14009 %}
14010 
14011 // This pattern is automatically generated from aarch64_ad.m4
14012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14013 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
14014 %{
14015   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
14016   ins_cost(1.9 * INSN_COST);
14017   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
14018 
14019    ins_encode %{
14020      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14021             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
14022    %}
14023   ins_pipe(ialu_reg_reg_shift);
14024 %}
14025 
14026 // This pattern is automatically generated from aarch64_ad.m4
14027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14028 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14029 %{
14030   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14031   ins_cost(1.9 * INSN_COST);
14032   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
14033 
14034    ins_encode %{
14035      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14036             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14037    %}
14038   ins_pipe(ialu_reg_reg_shift);
14039 %}
14040 
14041 // This pattern is automatically generated from aarch64_ad.m4
14042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14043 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14044 %{
14045   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14046   ins_cost(1.9 * INSN_COST);
14047   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
14048 
14049    ins_encode %{
14050      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14051             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14052    %}
14053   ins_pipe(ialu_reg_reg_shift);
14054 %}
14055 
14056 // This pattern is automatically generated from aarch64_ad.m4
14057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14058 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14059 %{
14060   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14061   ins_cost(1.9 * INSN_COST);
14062   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
14063 
14064    ins_encode %{
14065      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14066             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14067    %}
14068   ins_pipe(ialu_reg_reg_shift);
14069 %}
14070 
14071 // This pattern is automatically generated from aarch64_ad.m4
14072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14073 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14074 %{
14075   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14076   ins_cost(1.9 * INSN_COST);
14077   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
14078 
14079    ins_encode %{
14080      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14081             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14082    %}
14083   ins_pipe(ialu_reg_reg_shift);
14084 %}
14085 
14086 // This pattern is automatically generated from aarch64_ad.m4
14087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14088 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14089 %{
14090   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14091   ins_cost(1.9 * INSN_COST);
14092   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
14093 
14094    ins_encode %{
14095      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14096             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14097    %}
14098   ins_pipe(ialu_reg_reg_shift);
14099 %}
14100 
14101 // This pattern is automatically generated from aarch64_ad.m4
14102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14103 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14104 %{
14105   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14106   ins_cost(1.9 * INSN_COST);
14107   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
14108 
14109    ins_encode %{
14110      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14111             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14112    %}
14113   ins_pipe(ialu_reg_reg_shift);
14114 %}
14115 
14116 // This pattern is automatically generated from aarch64_ad.m4
14117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14118 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14119 %{
14120   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14121   ins_cost(1.9 * INSN_COST);
14122   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
14123 
14124    ins_encode %{
14125      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14126             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14127    %}
14128   ins_pipe(ialu_reg_reg_shift);
14129 %}
14130 
14131 // This pattern is automatically generated from aarch64_ad.m4
14132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14133 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14134 %{
14135   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14136   ins_cost(1.9 * INSN_COST);
14137   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
14138 
14139    ins_encode %{
14140      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14141             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14142    %}
14143   ins_pipe(ialu_reg_reg_shift);
14144 %}
14145 
14146 // This pattern is automatically generated from aarch64_ad.m4
14147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14148 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14149 %{
14150   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14151   ins_cost(1.9 * INSN_COST);
14152   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
14153 
14154    ins_encode %{
14155      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14156             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14157    %}
14158   ins_pipe(ialu_reg_reg_shift);
14159 %}
14160 
14161 // This pattern is automatically generated from aarch64_ad.m4
14162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14163 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14164 %{
14165   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14166   ins_cost(1.9 * INSN_COST);
14167   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
14168 
14169    ins_encode %{
14170      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14171             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14172    %}
14173   ins_pipe(ialu_reg_reg_shift);
14174 %}
14175 
14176 
14177 
14178 // END This section of the file is automatically generated. Do not edit --------------
14179 
14180 
14181 // ============================================================================
14182 // Floating Point Arithmetic Instructions
14183 
14184 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14185   match(Set dst (AddF src1 src2));
14186 
14187   ins_cost(INSN_COST * 5);
14188   format %{ "fadds   $dst, $src1, $src2" %}
14189 
14190   ins_encode %{
14191     __ fadds(as_FloatRegister($dst$$reg),
14192              as_FloatRegister($src1$$reg),
14193              as_FloatRegister($src2$$reg));
14194   %}
14195 
14196   ins_pipe(fp_dop_reg_reg_s);
14197 %}
14198 
14199 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14200   match(Set dst (AddD src1 src2));
14201 
14202   ins_cost(INSN_COST * 5);
14203   format %{ "faddd   $dst, $src1, $src2" %}
14204 
14205   ins_encode %{
14206     __ faddd(as_FloatRegister($dst$$reg),
14207              as_FloatRegister($src1$$reg),
14208              as_FloatRegister($src2$$reg));
14209   %}
14210 
14211   ins_pipe(fp_dop_reg_reg_d);
14212 %}
14213 
14214 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14215   match(Set dst (SubF src1 src2));
14216 
14217   ins_cost(INSN_COST * 5);
14218   format %{ "fsubs   $dst, $src1, $src2" %}
14219 
14220   ins_encode %{
14221     __ fsubs(as_FloatRegister($dst$$reg),
14222              as_FloatRegister($src1$$reg),
14223              as_FloatRegister($src2$$reg));
14224   %}
14225 
14226   ins_pipe(fp_dop_reg_reg_s);
14227 %}
14228 
14229 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14230   match(Set dst (SubD src1 src2));
14231 
14232   ins_cost(INSN_COST * 5);
14233   format %{ "fsubd   $dst, $src1, $src2" %}
14234 
14235   ins_encode %{
14236     __ fsubd(as_FloatRegister($dst$$reg),
14237              as_FloatRegister($src1$$reg),
14238              as_FloatRegister($src2$$reg));
14239   %}
14240 
14241   ins_pipe(fp_dop_reg_reg_d);
14242 %}
14243 
14244 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14245   match(Set dst (MulF src1 src2));
14246 
14247   ins_cost(INSN_COST * 6);
14248   format %{ "fmuls   $dst, $src1, $src2" %}
14249 
14250   ins_encode %{
14251     __ fmuls(as_FloatRegister($dst$$reg),
14252              as_FloatRegister($src1$$reg),
14253              as_FloatRegister($src2$$reg));
14254   %}
14255 
14256   ins_pipe(fp_dop_reg_reg_s);
14257 %}
14258 
14259 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14260   match(Set dst (MulD src1 src2));
14261 
14262   ins_cost(INSN_COST * 6);
14263   format %{ "fmuld   $dst, $src1, $src2" %}
14264 
14265   ins_encode %{
14266     __ fmuld(as_FloatRegister($dst$$reg),
14267              as_FloatRegister($src1$$reg),
14268              as_FloatRegister($src2$$reg));
14269   %}
14270 
14271   ins_pipe(fp_dop_reg_reg_d);
14272 %}
14273 
14274 // src1 * src2 + src3
14275 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14276   predicate(UseFMA);
14277   match(Set dst (FmaF src3 (Binary src1 src2)));
14278 
14279   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14280 
14281   ins_encode %{
14282     __ fmadds(as_FloatRegister($dst$$reg),
14283              as_FloatRegister($src1$$reg),
14284              as_FloatRegister($src2$$reg),
14285              as_FloatRegister($src3$$reg));
14286   %}
14287 
14288   ins_pipe(pipe_class_default);
14289 %}
14290 
14291 // src1 * src2 + src3
14292 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14293   predicate(UseFMA);
14294   match(Set dst (FmaD src3 (Binary src1 src2)));
14295 
14296   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14297 
14298   ins_encode %{
14299     __ fmaddd(as_FloatRegister($dst$$reg),
14300              as_FloatRegister($src1$$reg),
14301              as_FloatRegister($src2$$reg),
14302              as_FloatRegister($src3$$reg));
14303   %}
14304 
14305   ins_pipe(pipe_class_default);
14306 %}
14307 
14308 // -src1 * src2 + src3
14309 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14310   predicate(UseFMA);
14311   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14312   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14313 
14314   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14315 
14316   ins_encode %{
14317     __ fmsubs(as_FloatRegister($dst$$reg),
14318               as_FloatRegister($src1$$reg),
14319               as_FloatRegister($src2$$reg),
14320               as_FloatRegister($src3$$reg));
14321   %}
14322 
14323   ins_pipe(pipe_class_default);
14324 %}
14325 
14326 // -src1 * src2 + src3
14327 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14328   predicate(UseFMA);
14329   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14330   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14331 
14332   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14333 
14334   ins_encode %{
14335     __ fmsubd(as_FloatRegister($dst$$reg),
14336               as_FloatRegister($src1$$reg),
14337               as_FloatRegister($src2$$reg),
14338               as_FloatRegister($src3$$reg));
14339   %}
14340 
14341   ins_pipe(pipe_class_default);
14342 %}
14343 
14344 // -src1 * src2 - src3
14345 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14346   predicate(UseFMA);
14347   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14348   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14349 
14350   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14351 
14352   ins_encode %{
14353     __ fnmadds(as_FloatRegister($dst$$reg),
14354                as_FloatRegister($src1$$reg),
14355                as_FloatRegister($src2$$reg),
14356                as_FloatRegister($src3$$reg));
14357   %}
14358 
14359   ins_pipe(pipe_class_default);
14360 %}
14361 
14362 // -src1 * src2 - src3
14363 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14364   predicate(UseFMA);
14365   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14366   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14367 
14368   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14369 
14370   ins_encode %{
14371     __ fnmaddd(as_FloatRegister($dst$$reg),
14372                as_FloatRegister($src1$$reg),
14373                as_FloatRegister($src2$$reg),
14374                as_FloatRegister($src3$$reg));
14375   %}
14376 
14377   ins_pipe(pipe_class_default);
14378 %}
14379 
14380 // src1 * src2 - src3
14381 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14382   predicate(UseFMA);
14383   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14384 
14385   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14386 
14387   ins_encode %{
14388     __ fnmsubs(as_FloatRegister($dst$$reg),
14389                as_FloatRegister($src1$$reg),
14390                as_FloatRegister($src2$$reg),
14391                as_FloatRegister($src3$$reg));
14392   %}
14393 
14394   ins_pipe(pipe_class_default);
14395 %}
14396 
14397 // src1 * src2 - src3
14398 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14399   predicate(UseFMA);
14400   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14401 
14402   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14403 
14404   ins_encode %{
14405   // n.b. insn name should be fnmsubd
14406     __ fnmsub(as_FloatRegister($dst$$reg),
14407               as_FloatRegister($src1$$reg),
14408               as_FloatRegister($src2$$reg),
14409               as_FloatRegister($src3$$reg));
14410   %}
14411 
14412   ins_pipe(pipe_class_default);
14413 %}
14414 
14415 
14416 // Math.max(FF)F
14417 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14418   match(Set dst (MaxF src1 src2));
14419 
14420   format %{ "fmaxs   $dst, $src1, $src2" %}
14421   ins_encode %{
14422     __ fmaxs(as_FloatRegister($dst$$reg),
14423              as_FloatRegister($src1$$reg),
14424              as_FloatRegister($src2$$reg));
14425   %}
14426 
14427   ins_pipe(fp_dop_reg_reg_s);
14428 %}
14429 
14430 // Math.min(FF)F
14431 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14432   match(Set dst (MinF src1 src2));
14433 
14434   format %{ "fmins   $dst, $src1, $src2" %}
14435   ins_encode %{
14436     __ fmins(as_FloatRegister($dst$$reg),
14437              as_FloatRegister($src1$$reg),
14438              as_FloatRegister($src2$$reg));
14439   %}
14440 
14441   ins_pipe(fp_dop_reg_reg_s);
14442 %}
14443 
14444 // Math.max(DD)D
14445 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14446   match(Set dst (MaxD src1 src2));
14447 
14448   format %{ "fmaxd   $dst, $src1, $src2" %}
14449   ins_encode %{
14450     __ fmaxd(as_FloatRegister($dst$$reg),
14451              as_FloatRegister($src1$$reg),
14452              as_FloatRegister($src2$$reg));
14453   %}
14454 
14455   ins_pipe(fp_dop_reg_reg_d);
14456 %}
14457 
14458 // Math.min(DD)D
14459 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14460   match(Set dst (MinD src1 src2));
14461 
14462   format %{ "fmind   $dst, $src1, $src2" %}
14463   ins_encode %{
14464     __ fmind(as_FloatRegister($dst$$reg),
14465              as_FloatRegister($src1$$reg),
14466              as_FloatRegister($src2$$reg));
14467   %}
14468 
14469   ins_pipe(fp_dop_reg_reg_d);
14470 %}
14471 
14472 
14473 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14474   match(Set dst (DivF src1  src2));
14475 
14476   ins_cost(INSN_COST * 18);
14477   format %{ "fdivs   $dst, $src1, $src2" %}
14478 
14479   ins_encode %{
14480     __ fdivs(as_FloatRegister($dst$$reg),
14481              as_FloatRegister($src1$$reg),
14482              as_FloatRegister($src2$$reg));
14483   %}
14484 
14485   ins_pipe(fp_div_s);
14486 %}
14487 
14488 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14489   match(Set dst (DivD src1  src2));
14490 
14491   ins_cost(INSN_COST * 32);
14492   format %{ "fdivd   $dst, $src1, $src2" %}
14493 
14494   ins_encode %{
14495     __ fdivd(as_FloatRegister($dst$$reg),
14496              as_FloatRegister($src1$$reg),
14497              as_FloatRegister($src2$$reg));
14498   %}
14499 
14500   ins_pipe(fp_div_d);
14501 %}
14502 
14503 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14504   match(Set dst (NegF src));
14505 
14506   ins_cost(INSN_COST * 3);
14507   format %{ "fneg   $dst, $src" %}
14508 
14509   ins_encode %{
14510     __ fnegs(as_FloatRegister($dst$$reg),
14511              as_FloatRegister($src$$reg));
14512   %}
14513 
14514   ins_pipe(fp_uop_s);
14515 %}
14516 
14517 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14518   match(Set dst (NegD src));
14519 
14520   ins_cost(INSN_COST * 3);
14521   format %{ "fnegd   $dst, $src" %}
14522 
14523   ins_encode %{
14524     __ fnegd(as_FloatRegister($dst$$reg),
14525              as_FloatRegister($src$$reg));
14526   %}
14527 
14528   ins_pipe(fp_uop_d);
14529 %}
14530 
14531 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14532 %{
14533   match(Set dst (AbsI src));
14534 
14535   effect(KILL cr);
14536   ins_cost(INSN_COST * 2);
14537   format %{ "cmpw  $src, zr\n\t"
14538             "cnegw $dst, $src, Assembler::LT\t# int abs"
14539   %}
14540 
14541   ins_encode %{
14542     __ cmpw(as_Register($src$$reg), zr);
14543     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14544   %}
14545   ins_pipe(pipe_class_default);
14546 %}
14547 
14548 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14549 %{
14550   match(Set dst (AbsL src));
14551 
14552   effect(KILL cr);
14553   ins_cost(INSN_COST * 2);
14554   format %{ "cmp  $src, zr\n\t"
14555             "cneg $dst, $src, Assembler::LT\t# long abs"
14556   %}
14557 
14558   ins_encode %{
14559     __ cmp(as_Register($src$$reg), zr);
14560     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14561   %}
14562   ins_pipe(pipe_class_default);
14563 %}
14564 
14565 instruct absF_reg(vRegF dst, vRegF src) %{
14566   match(Set dst (AbsF src));
14567 
14568   ins_cost(INSN_COST * 3);
14569   format %{ "fabss   $dst, $src" %}
14570   ins_encode %{
14571     __ fabss(as_FloatRegister($dst$$reg),
14572              as_FloatRegister($src$$reg));
14573   %}
14574 
14575   ins_pipe(fp_uop_s);
14576 %}
14577 
14578 instruct absD_reg(vRegD dst, vRegD src) %{
14579   match(Set dst (AbsD src));
14580 
14581   ins_cost(INSN_COST * 3);
14582   format %{ "fabsd   $dst, $src" %}
14583   ins_encode %{
14584     __ fabsd(as_FloatRegister($dst$$reg),
14585              as_FloatRegister($src$$reg));
14586   %}
14587 
14588   ins_pipe(fp_uop_d);
14589 %}
14590 
14591 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14592   match(Set dst (AbsF (SubF src1 src2)));
14593 
14594   ins_cost(INSN_COST * 3);
14595   format %{ "fabds   $dst, $src1, $src2" %}
14596   ins_encode %{
14597     __ fabds(as_FloatRegister($dst$$reg),
14598              as_FloatRegister($src1$$reg),
14599              as_FloatRegister($src2$$reg));
14600   %}
14601 
14602   ins_pipe(fp_uop_s);
14603 %}
14604 
14605 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14606   match(Set dst (AbsD (SubD src1 src2)));
14607 
14608   ins_cost(INSN_COST * 3);
14609   format %{ "fabdd   $dst, $src1, $src2" %}
14610   ins_encode %{
14611     __ fabdd(as_FloatRegister($dst$$reg),
14612              as_FloatRegister($src1$$reg),
14613              as_FloatRegister($src2$$reg));
14614   %}
14615 
14616   ins_pipe(fp_uop_d);
14617 %}
14618 
14619 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14620   match(Set dst (SqrtD src));
14621 
14622   ins_cost(INSN_COST * 50);
14623   format %{ "fsqrtd  $dst, $src" %}
14624   ins_encode %{
14625     __ fsqrtd(as_FloatRegister($dst$$reg),
14626              as_FloatRegister($src$$reg));
14627   %}
14628 
14629   ins_pipe(fp_div_s);
14630 %}
14631 
14632 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14633   match(Set dst (SqrtF src));
14634 
14635   ins_cost(INSN_COST * 50);
14636   format %{ "fsqrts  $dst, $src" %}
14637   ins_encode %{
14638     __ fsqrts(as_FloatRegister($dst$$reg),
14639              as_FloatRegister($src$$reg));
14640   %}
14641 
14642   ins_pipe(fp_div_d);
14643 %}
14644 
14645 // Math.rint, floor, ceil
14646 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14647   match(Set dst (RoundDoubleMode src rmode));
14648   format %{ "frint  $dst, $src, $rmode" %}
14649   ins_encode %{
14650     switch ($rmode$$constant) {
14651       case RoundDoubleModeNode::rmode_rint:
14652         __ frintnd(as_FloatRegister($dst$$reg),
14653                    as_FloatRegister($src$$reg));
14654         break;
14655       case RoundDoubleModeNode::rmode_floor:
14656         __ frintmd(as_FloatRegister($dst$$reg),
14657                    as_FloatRegister($src$$reg));
14658         break;
14659       case RoundDoubleModeNode::rmode_ceil:
14660         __ frintpd(as_FloatRegister($dst$$reg),
14661                    as_FloatRegister($src$$reg));
14662         break;
14663     }
14664   %}
14665   ins_pipe(fp_uop_d);
14666 %}
14667 
14668 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14669   match(Set dst (CopySignD src1 (Binary src2 zero)));
14670   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14671   format %{ "CopySignD  $dst $src1 $src2" %}
14672   ins_encode %{
14673     FloatRegister dst = as_FloatRegister($dst$$reg),
14674                   src1 = as_FloatRegister($src1$$reg),
14675                   src2 = as_FloatRegister($src2$$reg),
14676                   zero = as_FloatRegister($zero$$reg);
14677     __ fnegd(dst, zero);
14678     __ bsl(dst, __ T8B, src2, src1);
14679   %}
14680   ins_pipe(fp_uop_d);
14681 %}
14682 
14683 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14684   match(Set dst (CopySignF src1 src2));
14685   effect(TEMP_DEF dst, USE src1, USE src2);
14686   format %{ "CopySignF  $dst $src1 $src2" %}
14687   ins_encode %{
14688     FloatRegister dst = as_FloatRegister($dst$$reg),
14689                   src1 = as_FloatRegister($src1$$reg),
14690                   src2 = as_FloatRegister($src2$$reg);
14691     __ movi(dst, __ T2S, 0x80, 24);
14692     __ bsl(dst, __ T8B, src2, src1);
14693   %}
14694   ins_pipe(fp_uop_d);
14695 %}
14696 
14697 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14698   match(Set dst (SignumD src (Binary zero one)));
14699   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14700   format %{ "signumD  $dst, $src" %}
14701   ins_encode %{
14702     FloatRegister src = as_FloatRegister($src$$reg),
14703                   dst = as_FloatRegister($dst$$reg),
14704                   zero = as_FloatRegister($zero$$reg),
14705                   one = as_FloatRegister($one$$reg);
14706     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14707     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14708     // Bit selection instruction gets bit from "one" for each enabled bit in
14709     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14710     // NaN the whole "src" will be copied because "dst" is zero. For all other
14711     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14712     // from "src", and all other bits are copied from 1.0.
14713     __ bsl(dst, __ T8B, one, src);
14714   %}
14715   ins_pipe(fp_uop_d);
14716 %}
14717 
14718 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14719   match(Set dst (SignumF src (Binary zero one)));
14720   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14721   format %{ "signumF  $dst, $src" %}
14722   ins_encode %{
14723     FloatRegister src = as_FloatRegister($src$$reg),
14724                   dst = as_FloatRegister($dst$$reg),
14725                   zero = as_FloatRegister($zero$$reg),
14726                   one = as_FloatRegister($one$$reg);
14727     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14728     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14729     // Bit selection instruction gets bit from "one" for each enabled bit in
14730     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14731     // NaN the whole "src" will be copied because "dst" is zero. For all other
14732     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14733     // from "src", and all other bits are copied from 1.0.
14734     __ bsl(dst, __ T8B, one, src);
14735   %}
14736   ins_pipe(fp_uop_d);
14737 %}
14738 
14739 instruct onspinwait() %{
14740   match(OnSpinWait);
14741   ins_cost(INSN_COST);
14742 
14743   format %{ "onspinwait" %}
14744 
14745   ins_encode %{
14746     __ spin_wait();
14747   %}
14748   ins_pipe(pipe_class_empty);
14749 %}
14750 
14751 // ============================================================================
14752 // Logical Instructions
14753 
14754 // Integer Logical Instructions
14755 
14756 // And Instructions
14757 
14758 
14759 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14760   match(Set dst (AndI src1 src2));
14761 
14762   format %{ "andw  $dst, $src1, $src2\t# int" %}
14763 
14764   ins_cost(INSN_COST);
14765   ins_encode %{
14766     __ andw(as_Register($dst$$reg),
14767             as_Register($src1$$reg),
14768             as_Register($src2$$reg));
14769   %}
14770 
14771   ins_pipe(ialu_reg_reg);
14772 %}
14773 
14774 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14775   match(Set dst (AndI src1 src2));
14776 
14777   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14778 
14779   ins_cost(INSN_COST);
14780   ins_encode %{
14781     __ andw(as_Register($dst$$reg),
14782             as_Register($src1$$reg),
14783             (uint64_t)($src2$$constant));
14784   %}
14785 
14786   ins_pipe(ialu_reg_imm);
14787 %}
14788 
14789 // Or Instructions
14790 
14791 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14792   match(Set dst (OrI src1 src2));
14793 
14794   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14795 
14796   ins_cost(INSN_COST);
14797   ins_encode %{
14798     __ orrw(as_Register($dst$$reg),
14799             as_Register($src1$$reg),
14800             as_Register($src2$$reg));
14801   %}
14802 
14803   ins_pipe(ialu_reg_reg);
14804 %}
14805 
14806 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14807   match(Set dst (OrI src1 src2));
14808 
14809   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14810 
14811   ins_cost(INSN_COST);
14812   ins_encode %{
14813     __ orrw(as_Register($dst$$reg),
14814             as_Register($src1$$reg),
14815             (uint64_t)($src2$$constant));
14816   %}
14817 
14818   ins_pipe(ialu_reg_imm);
14819 %}
14820 
14821 // Xor Instructions
14822 
14823 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14824   match(Set dst (XorI src1 src2));
14825 
14826   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14827 
14828   ins_cost(INSN_COST);
14829   ins_encode %{
14830     __ eorw(as_Register($dst$$reg),
14831             as_Register($src1$$reg),
14832             as_Register($src2$$reg));
14833   %}
14834 
14835   ins_pipe(ialu_reg_reg);
14836 %}
14837 
14838 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14839   match(Set dst (XorI src1 src2));
14840 
14841   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14842 
14843   ins_cost(INSN_COST);
14844   ins_encode %{
14845     __ eorw(as_Register($dst$$reg),
14846             as_Register($src1$$reg),
14847             (uint64_t)($src2$$constant));
14848   %}
14849 
14850   ins_pipe(ialu_reg_imm);
14851 %}
14852 
14853 // Long Logical Instructions
14854 // TODO
14855 
14856 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14857   match(Set dst (AndL src1 src2));
14858 
14859   format %{ "and  $dst, $src1, $src2\t# int" %}
14860 
14861   ins_cost(INSN_COST);
14862   ins_encode %{
14863     __ andr(as_Register($dst$$reg),
14864             as_Register($src1$$reg),
14865             as_Register($src2$$reg));
14866   %}
14867 
14868   ins_pipe(ialu_reg_reg);
14869 %}
14870 
14871 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14872   match(Set dst (AndL src1 src2));
14873 
14874   format %{ "and  $dst, $src1, $src2\t# int" %}
14875 
14876   ins_cost(INSN_COST);
14877   ins_encode %{
14878     __ andr(as_Register($dst$$reg),
14879             as_Register($src1$$reg),
14880             (uint64_t)($src2$$constant));
14881   %}
14882 
14883   ins_pipe(ialu_reg_imm);
14884 %}
14885 
14886 // Or Instructions
14887 
14888 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14889   match(Set dst (OrL src1 src2));
14890 
14891   format %{ "orr  $dst, $src1, $src2\t# int" %}
14892 
14893   ins_cost(INSN_COST);
14894   ins_encode %{
14895     __ orr(as_Register($dst$$reg),
14896            as_Register($src1$$reg),
14897            as_Register($src2$$reg));
14898   %}
14899 
14900   ins_pipe(ialu_reg_reg);
14901 %}
14902 
14903 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14904   match(Set dst (OrL src1 src2));
14905 
14906   format %{ "orr  $dst, $src1, $src2\t# int" %}
14907 
14908   ins_cost(INSN_COST);
14909   ins_encode %{
14910     __ orr(as_Register($dst$$reg),
14911            as_Register($src1$$reg),
14912            (uint64_t)($src2$$constant));
14913   %}
14914 
14915   ins_pipe(ialu_reg_imm);
14916 %}
14917 
14918 // Xor Instructions
14919 
14920 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14921   match(Set dst (XorL src1 src2));
14922 
14923   format %{ "eor  $dst, $src1, $src2\t# int" %}
14924 
14925   ins_cost(INSN_COST);
14926   ins_encode %{
14927     __ eor(as_Register($dst$$reg),
14928            as_Register($src1$$reg),
14929            as_Register($src2$$reg));
14930   %}
14931 
14932   ins_pipe(ialu_reg_reg);
14933 %}
14934 
14935 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14936   match(Set dst (XorL src1 src2));
14937 
14938   ins_cost(INSN_COST);
14939   format %{ "eor  $dst, $src1, $src2\t# int" %}
14940 
14941   ins_encode %{
14942     __ eor(as_Register($dst$$reg),
14943            as_Register($src1$$reg),
14944            (uint64_t)($src2$$constant));
14945   %}
14946 
14947   ins_pipe(ialu_reg_imm);
14948 %}
14949 
14950 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14951 %{
14952   match(Set dst (ConvI2L src));
14953 
14954   ins_cost(INSN_COST);
14955   format %{ "sxtw  $dst, $src\t# i2l" %}
14956   ins_encode %{
14957     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14958   %}
14959   ins_pipe(ialu_reg_shift);
14960 %}
14961 
14962 // this pattern occurs in bigmath arithmetic
14963 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14964 %{
14965   match(Set dst (AndL (ConvI2L src) mask));
14966 
14967   ins_cost(INSN_COST);
14968   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14969   ins_encode %{
14970     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14971   %}
14972 
14973   ins_pipe(ialu_reg_shift);
14974 %}
14975 
14976 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14977   match(Set dst (ConvL2I src));
14978 
14979   ins_cost(INSN_COST);
14980   format %{ "movw  $dst, $src \t// l2i" %}
14981 
14982   ins_encode %{
14983     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14984   %}
14985 
14986   ins_pipe(ialu_reg);
14987 %}
14988 
14989 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14990 %{
14991   match(Set dst (Conv2B src));
14992   effect(KILL cr);
14993 
14994   format %{
14995     "cmpw $src, zr\n\t"
14996     "cset $dst, ne"
14997   %}
14998 
14999   ins_encode %{
15000     __ cmpw(as_Register($src$$reg), zr);
15001     __ cset(as_Register($dst$$reg), Assembler::NE);
15002   %}
15003 
15004   ins_pipe(ialu_reg);
15005 %}
15006 
15007 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
15008 %{
15009   match(Set dst (Conv2B src));
15010   effect(KILL cr);
15011 
15012   format %{
15013     "cmp  $src, zr\n\t"
15014     "cset $dst, ne"
15015   %}
15016 
15017   ins_encode %{
15018     __ cmp(as_Register($src$$reg), zr);
15019     __ cset(as_Register($dst$$reg), Assembler::NE);
15020   %}
15021 
15022   ins_pipe(ialu_reg);
15023 %}
15024 
15025 instruct convD2F_reg(vRegF dst, vRegD src) %{
15026   match(Set dst (ConvD2F src));
15027 
15028   ins_cost(INSN_COST * 5);
15029   format %{ "fcvtd  $dst, $src \t// d2f" %}
15030 
15031   ins_encode %{
15032     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15033   %}
15034 
15035   ins_pipe(fp_d2f);
15036 %}
15037 
15038 instruct convF2D_reg(vRegD dst, vRegF src) %{
15039   match(Set dst (ConvF2D src));
15040 
15041   ins_cost(INSN_COST * 5);
15042   format %{ "fcvts  $dst, $src \t// f2d" %}
15043 
15044   ins_encode %{
15045     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15046   %}
15047 
15048   ins_pipe(fp_f2d);
15049 %}
15050 
15051 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15052   match(Set dst (ConvF2I src));
15053 
15054   ins_cost(INSN_COST * 5);
15055   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
15056 
15057   ins_encode %{
15058     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15059   %}
15060 
15061   ins_pipe(fp_f2i);
15062 %}
15063 
15064 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15065   match(Set dst (ConvF2L src));
15066 
15067   ins_cost(INSN_COST * 5);
15068   format %{ "fcvtzs  $dst, $src \t// f2l" %}
15069 
15070   ins_encode %{
15071     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15072   %}
15073 
15074   ins_pipe(fp_f2l);
15075 %}
15076 
15077 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15078   match(Set dst (ConvI2F src));
15079 
15080   ins_cost(INSN_COST * 5);
15081   format %{ "scvtfws  $dst, $src \t// i2f" %}
15082 
15083   ins_encode %{
15084     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15085   %}
15086 
15087   ins_pipe(fp_i2f);
15088 %}
15089 
15090 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15091   match(Set dst (ConvL2F src));
15092 
15093   ins_cost(INSN_COST * 5);
15094   format %{ "scvtfs  $dst, $src \t// l2f" %}
15095 
15096   ins_encode %{
15097     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15098   %}
15099 
15100   ins_pipe(fp_l2f);
15101 %}
15102 
15103 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15104   match(Set dst (ConvD2I src));
15105 
15106   ins_cost(INSN_COST * 5);
15107   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15108 
15109   ins_encode %{
15110     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15111   %}
15112 
15113   ins_pipe(fp_d2i);
15114 %}
15115 
15116 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15117   match(Set dst (ConvD2L src));
15118 
15119   ins_cost(INSN_COST * 5);
15120   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15121 
15122   ins_encode %{
15123     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15124   %}
15125 
15126   ins_pipe(fp_d2l);
15127 %}
15128 
15129 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15130   match(Set dst (ConvI2D src));
15131 
15132   ins_cost(INSN_COST * 5);
15133   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15134 
15135   ins_encode %{
15136     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15137   %}
15138 
15139   ins_pipe(fp_i2d);
15140 %}
15141 
15142 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15143   match(Set dst (ConvL2D src));
15144 
15145   ins_cost(INSN_COST * 5);
15146   format %{ "scvtfd  $dst, $src \t// l2d" %}
15147 
15148   ins_encode %{
15149     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15150   %}
15151 
15152   ins_pipe(fp_l2d);
15153 %}
15154 
15155 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15156 %{
15157   match(Set dst (RoundD src));
15158   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15159   format %{ "java_round_double $dst,$src"%}
15160   ins_encode %{
15161     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15162                          as_FloatRegister($ftmp$$reg));
15163   %}
15164   ins_pipe(pipe_slow);
15165 %}
15166 
15167 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15168 %{
15169   match(Set dst (RoundF src));
15170   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15171   format %{ "java_round_float $dst,$src"%}
15172   ins_encode %{
15173     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15174                         as_FloatRegister($ftmp$$reg));
15175   %}
15176   ins_pipe(pipe_slow);
15177 %}
15178 
15179 // stack <-> reg and reg <-> reg shuffles with no conversion
15180 
15181 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15182 
15183   match(Set dst (MoveF2I src));
15184 
15185   effect(DEF dst, USE src);
15186 
15187   ins_cost(4 * INSN_COST);
15188 
15189   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15190 
15191   ins_encode %{
15192     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15193   %}
15194 
15195   ins_pipe(iload_reg_reg);
15196 
15197 %}
15198 
15199 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15200 
15201   match(Set dst (MoveI2F src));
15202 
15203   effect(DEF dst, USE src);
15204 
15205   ins_cost(4 * INSN_COST);
15206 
15207   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15208 
15209   ins_encode %{
15210     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15211   %}
15212 
15213   ins_pipe(pipe_class_memory);
15214 
15215 %}
15216 
15217 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15218 
15219   match(Set dst (MoveD2L src));
15220 
15221   effect(DEF dst, USE src);
15222 
15223   ins_cost(4 * INSN_COST);
15224 
15225   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15226 
15227   ins_encode %{
15228     __ ldr($dst$$Register, Address(sp, $src$$disp));
15229   %}
15230 
15231   ins_pipe(iload_reg_reg);
15232 
15233 %}
15234 
15235 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15236 
15237   match(Set dst (MoveL2D src));
15238 
15239   effect(DEF dst, USE src);
15240 
15241   ins_cost(4 * INSN_COST);
15242 
15243   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15244 
15245   ins_encode %{
15246     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15247   %}
15248 
15249   ins_pipe(pipe_class_memory);
15250 
15251 %}
15252 
15253 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15254 
15255   match(Set dst (MoveF2I src));
15256 
15257   effect(DEF dst, USE src);
15258 
15259   ins_cost(INSN_COST);
15260 
15261   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15262 
15263   ins_encode %{
15264     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15265   %}
15266 
15267   ins_pipe(pipe_class_memory);
15268 
15269 %}
15270 
15271 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15272 
15273   match(Set dst (MoveI2F src));
15274 
15275   effect(DEF dst, USE src);
15276 
15277   ins_cost(INSN_COST);
15278 
15279   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15280 
15281   ins_encode %{
15282     __ strw($src$$Register, Address(sp, $dst$$disp));
15283   %}
15284 
15285   ins_pipe(istore_reg_reg);
15286 
15287 %}
15288 
15289 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15290 
15291   match(Set dst (MoveD2L src));
15292 
15293   effect(DEF dst, USE src);
15294 
15295   ins_cost(INSN_COST);
15296 
15297   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15298 
15299   ins_encode %{
15300     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15301   %}
15302 
15303   ins_pipe(pipe_class_memory);
15304 
15305 %}
15306 
15307 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15308 
15309   match(Set dst (MoveL2D src));
15310 
15311   effect(DEF dst, USE src);
15312 
15313   ins_cost(INSN_COST);
15314 
15315   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15316 
15317   ins_encode %{
15318     __ str($src$$Register, Address(sp, $dst$$disp));
15319   %}
15320 
15321   ins_pipe(istore_reg_reg);
15322 
15323 %}
15324 
15325 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15326 
15327   match(Set dst (MoveF2I src));
15328 
15329   effect(DEF dst, USE src);
15330 
15331   ins_cost(INSN_COST);
15332 
15333   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15334 
15335   ins_encode %{
15336     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15337   %}
15338 
15339   ins_pipe(fp_f2i);
15340 
15341 %}
15342 
15343 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15344 
15345   match(Set dst (MoveI2F src));
15346 
15347   effect(DEF dst, USE src);
15348 
15349   ins_cost(INSN_COST);
15350 
15351   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15352 
15353   ins_encode %{
15354     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15355   %}
15356 
15357   ins_pipe(fp_i2f);
15358 
15359 %}
15360 
15361 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15362 
15363   match(Set dst (MoveD2L src));
15364 
15365   effect(DEF dst, USE src);
15366 
15367   ins_cost(INSN_COST);
15368 
15369   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15370 
15371   ins_encode %{
15372     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15373   %}
15374 
15375   ins_pipe(fp_d2l);
15376 
15377 %}
15378 
15379 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15380 
15381   match(Set dst (MoveL2D src));
15382 
15383   effect(DEF dst, USE src);
15384 
15385   ins_cost(INSN_COST);
15386 
15387   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15388 
15389   ins_encode %{
15390     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15391   %}
15392 
15393   ins_pipe(fp_l2d);
15394 
15395 %}
15396 
15397 // ============================================================================
15398 // clearing of an array
15399 
15400 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15401 %{
15402   match(Set dummy (ClearArray cnt base));
15403   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15404 
15405   ins_cost(4 * INSN_COST);
15406   format %{ "ClearArray $cnt, $base" %}
15407 
15408   ins_encode %{
15409     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15410     if (tpc == NULL) {
15411       ciEnv::current()->record_failure("CodeCache is full");
15412       return;
15413     }
15414   %}
15415 
15416   ins_pipe(pipe_class_memory);
15417 %}
15418 
15419 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15420 %{
15421   predicate((uint64_t)n->in(2)->get_long()
15422             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15423   match(Set dummy (ClearArray cnt base));
15424   effect(TEMP temp, USE_KILL base, KILL cr);
15425 
15426   ins_cost(4 * INSN_COST);
15427   format %{ "ClearArray $cnt, $base" %}
15428 
15429   ins_encode %{
15430     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15431   %}
15432 
15433   ins_pipe(pipe_class_memory);
15434 %}
15435 
15436 // ============================================================================
15437 // Overflow Math Instructions
15438 
15439 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15440 %{
15441   match(Set cr (OverflowAddI op1 op2));
15442 
15443   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15444   ins_cost(INSN_COST);
15445   ins_encode %{
15446     __ cmnw($op1$$Register, $op2$$Register);
15447   %}
15448 
15449   ins_pipe(icmp_reg_reg);
15450 %}
15451 
15452 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15453 %{
15454   match(Set cr (OverflowAddI op1 op2));
15455 
15456   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15457   ins_cost(INSN_COST);
15458   ins_encode %{
15459     __ cmnw($op1$$Register, $op2$$constant);
15460   %}
15461 
15462   ins_pipe(icmp_reg_imm);
15463 %}
15464 
15465 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15466 %{
15467   match(Set cr (OverflowAddL op1 op2));
15468 
15469   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15470   ins_cost(INSN_COST);
15471   ins_encode %{
15472     __ cmn($op1$$Register, $op2$$Register);
15473   %}
15474 
15475   ins_pipe(icmp_reg_reg);
15476 %}
15477 
15478 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15479 %{
15480   match(Set cr (OverflowAddL op1 op2));
15481 
15482   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15483   ins_cost(INSN_COST);
15484   ins_encode %{
15485     __ cmn($op1$$Register, $op2$$constant);
15486   %}
15487 
15488   ins_pipe(icmp_reg_imm);
15489 %}
15490 
15491 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15492 %{
15493   match(Set cr (OverflowSubI op1 op2));
15494 
15495   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15496   ins_cost(INSN_COST);
15497   ins_encode %{
15498     __ cmpw($op1$$Register, $op2$$Register);
15499   %}
15500 
15501   ins_pipe(icmp_reg_reg);
15502 %}
15503 
15504 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15505 %{
15506   match(Set cr (OverflowSubI op1 op2));
15507 
15508   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15509   ins_cost(INSN_COST);
15510   ins_encode %{
15511     __ cmpw($op1$$Register, $op2$$constant);
15512   %}
15513 
15514   ins_pipe(icmp_reg_imm);
15515 %}
15516 
15517 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15518 %{
15519   match(Set cr (OverflowSubL op1 op2));
15520 
15521   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15522   ins_cost(INSN_COST);
15523   ins_encode %{
15524     __ cmp($op1$$Register, $op2$$Register);
15525   %}
15526 
15527   ins_pipe(icmp_reg_reg);
15528 %}
15529 
15530 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15531 %{
15532   match(Set cr (OverflowSubL op1 op2));
15533 
15534   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15535   ins_cost(INSN_COST);
15536   ins_encode %{
15537     __ subs(zr, $op1$$Register, $op2$$constant);
15538   %}
15539 
15540   ins_pipe(icmp_reg_imm);
15541 %}
15542 
15543 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15544 %{
15545   match(Set cr (OverflowSubI zero op1));
15546 
15547   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15548   ins_cost(INSN_COST);
15549   ins_encode %{
15550     __ cmpw(zr, $op1$$Register);
15551   %}
15552 
15553   ins_pipe(icmp_reg_imm);
15554 %}
15555 
15556 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15557 %{
15558   match(Set cr (OverflowSubL zero op1));
15559 
15560   format %{ "cmp   zr, $op1\t# overflow check long" %}
15561   ins_cost(INSN_COST);
15562   ins_encode %{
15563     __ cmp(zr, $op1$$Register);
15564   %}
15565 
15566   ins_pipe(icmp_reg_imm);
15567 %}
15568 
15569 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15570 %{
15571   match(Set cr (OverflowMulI op1 op2));
15572 
15573   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15574             "cmp   rscratch1, rscratch1, sxtw\n\t"
15575             "movw  rscratch1, #0x80000000\n\t"
15576             "cselw rscratch1, rscratch1, zr, NE\n\t"
15577             "cmpw  rscratch1, #1" %}
15578   ins_cost(5 * INSN_COST);
15579   ins_encode %{
15580     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15581     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15582     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15583     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15584     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15585   %}
15586 
15587   ins_pipe(pipe_slow);
15588 %}
15589 
15590 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15591 %{
15592   match(If cmp (OverflowMulI op1 op2));
15593   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15594             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15595   effect(USE labl, KILL cr);
15596 
15597   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15598             "cmp   rscratch1, rscratch1, sxtw\n\t"
15599             "b$cmp   $labl" %}
15600   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15601   ins_encode %{
15602     Label* L = $labl$$label;
15603     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15604     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15605     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15606     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15607   %}
15608 
15609   ins_pipe(pipe_serial);
15610 %}
15611 
15612 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15613 %{
15614   match(Set cr (OverflowMulL op1 op2));
15615 
15616   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15617             "smulh rscratch2, $op1, $op2\n\t"
15618             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15619             "movw  rscratch1, #0x80000000\n\t"
15620             "cselw rscratch1, rscratch1, zr, NE\n\t"
15621             "cmpw  rscratch1, #1" %}
15622   ins_cost(6 * INSN_COST);
15623   ins_encode %{
15624     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15625     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15626     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15627     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15628     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15629     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15630   %}
15631 
15632   ins_pipe(pipe_slow);
15633 %}
15634 
15635 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15636 %{
15637   match(If cmp (OverflowMulL op1 op2));
15638   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15639             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15640   effect(USE labl, KILL cr);
15641 
15642   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15643             "smulh rscratch2, $op1, $op2\n\t"
15644             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15645             "b$cmp $labl" %}
15646   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15647   ins_encode %{
15648     Label* L = $labl$$label;
15649     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15650     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15651     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15652     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15653     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15654   %}
15655 
15656   ins_pipe(pipe_serial);
15657 %}
15658 
15659 // ============================================================================
15660 // Compare Instructions
15661 
15662 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15663 %{
15664   match(Set cr (CmpI op1 op2));
15665 
15666   effect(DEF cr, USE op1, USE op2);
15667 
15668   ins_cost(INSN_COST);
15669   format %{ "cmpw  $op1, $op2" %}
15670 
15671   ins_encode(aarch64_enc_cmpw(op1, op2));
15672 
15673   ins_pipe(icmp_reg_reg);
15674 %}
15675 
15676 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15677 %{
15678   match(Set cr (CmpI op1 zero));
15679 
15680   effect(DEF cr, USE op1);
15681 
15682   ins_cost(INSN_COST);
15683   format %{ "cmpw $op1, 0" %}
15684 
15685   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15686 
15687   ins_pipe(icmp_reg_imm);
15688 %}
15689 
15690 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15691 %{
15692   match(Set cr (CmpI op1 op2));
15693 
15694   effect(DEF cr, USE op1);
15695 
15696   ins_cost(INSN_COST);
15697   format %{ "cmpw  $op1, $op2" %}
15698 
15699   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15700 
15701   ins_pipe(icmp_reg_imm);
15702 %}
15703 
15704 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15705 %{
15706   match(Set cr (CmpI op1 op2));
15707 
15708   effect(DEF cr, USE op1);
15709 
15710   ins_cost(INSN_COST * 2);
15711   format %{ "cmpw  $op1, $op2" %}
15712 
15713   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15714 
15715   ins_pipe(icmp_reg_imm);
15716 %}
15717 
15718 // Unsigned compare Instructions; really, same as signed compare
15719 // except it should only be used to feed an If or a CMovI which takes a
15720 // cmpOpU.
15721 
15722 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15723 %{
15724   match(Set cr (CmpU op1 op2));
15725 
15726   effect(DEF cr, USE op1, USE op2);
15727 
15728   ins_cost(INSN_COST);
15729   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15730 
15731   ins_encode(aarch64_enc_cmpw(op1, op2));
15732 
15733   ins_pipe(icmp_reg_reg);
15734 %}
15735 
15736 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15737 %{
15738   match(Set cr (CmpU op1 zero));
15739 
15740   effect(DEF cr, USE op1);
15741 
15742   ins_cost(INSN_COST);
15743   format %{ "cmpw $op1, #0\t# unsigned" %}
15744 
15745   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15746 
15747   ins_pipe(icmp_reg_imm);
15748 %}
15749 
15750 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15751 %{
15752   match(Set cr (CmpU op1 op2));
15753 
15754   effect(DEF cr, USE op1);
15755 
15756   ins_cost(INSN_COST);
15757   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15758 
15759   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15760 
15761   ins_pipe(icmp_reg_imm);
15762 %}
15763 
15764 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15765 %{
15766   match(Set cr (CmpU op1 op2));
15767 
15768   effect(DEF cr, USE op1);
15769 
15770   ins_cost(INSN_COST * 2);
15771   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15772 
15773   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15774 
15775   ins_pipe(icmp_reg_imm);
15776 %}
15777 
15778 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15779 %{
15780   match(Set cr (CmpL op1 op2));
15781 
15782   effect(DEF cr, USE op1, USE op2);
15783 
15784   ins_cost(INSN_COST);
15785   format %{ "cmp  $op1, $op2" %}
15786 
15787   ins_encode(aarch64_enc_cmp(op1, op2));
15788 
15789   ins_pipe(icmp_reg_reg);
15790 %}
15791 
15792 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15793 %{
15794   match(Set cr (CmpL op1 zero));
15795 
15796   effect(DEF cr, USE op1);
15797 
15798   ins_cost(INSN_COST);
15799   format %{ "tst  $op1" %}
15800 
15801   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15802 
15803   ins_pipe(icmp_reg_imm);
15804 %}
15805 
15806 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15807 %{
15808   match(Set cr (CmpL op1 op2));
15809 
15810   effect(DEF cr, USE op1);
15811 
15812   ins_cost(INSN_COST);
15813   format %{ "cmp  $op1, $op2" %}
15814 
15815   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15816 
15817   ins_pipe(icmp_reg_imm);
15818 %}
15819 
15820 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15821 %{
15822   match(Set cr (CmpL op1 op2));
15823 
15824   effect(DEF cr, USE op1);
15825 
15826   ins_cost(INSN_COST * 2);
15827   format %{ "cmp  $op1, $op2" %}
15828 
15829   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15830 
15831   ins_pipe(icmp_reg_imm);
15832 %}
15833 
15834 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15835 %{
15836   match(Set cr (CmpUL op1 op2));
15837 
15838   effect(DEF cr, USE op1, USE op2);
15839 
15840   ins_cost(INSN_COST);
15841   format %{ "cmp  $op1, $op2" %}
15842 
15843   ins_encode(aarch64_enc_cmp(op1, op2));
15844 
15845   ins_pipe(icmp_reg_reg);
15846 %}
15847 
15848 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15849 %{
15850   match(Set cr (CmpUL op1 zero));
15851 
15852   effect(DEF cr, USE op1);
15853 
15854   ins_cost(INSN_COST);
15855   format %{ "tst  $op1" %}
15856 
15857   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15858 
15859   ins_pipe(icmp_reg_imm);
15860 %}
15861 
15862 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15863 %{
15864   match(Set cr (CmpUL op1 op2));
15865 
15866   effect(DEF cr, USE op1);
15867 
15868   ins_cost(INSN_COST);
15869   format %{ "cmp  $op1, $op2" %}
15870 
15871   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15872 
15873   ins_pipe(icmp_reg_imm);
15874 %}
15875 
15876 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15877 %{
15878   match(Set cr (CmpUL op1 op2));
15879 
15880   effect(DEF cr, USE op1);
15881 
15882   ins_cost(INSN_COST * 2);
15883   format %{ "cmp  $op1, $op2" %}
15884 
15885   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15886 
15887   ins_pipe(icmp_reg_imm);
15888 %}
15889 
15890 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15891 %{
15892   match(Set cr (CmpP op1 op2));
15893 
15894   effect(DEF cr, USE op1, USE op2);
15895 
15896   ins_cost(INSN_COST);
15897   format %{ "cmp  $op1, $op2\t // ptr" %}
15898 
15899   ins_encode(aarch64_enc_cmpp(op1, op2));
15900 
15901   ins_pipe(icmp_reg_reg);
15902 %}
15903 
15904 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15905 %{
15906   match(Set cr (CmpN op1 op2));
15907 
15908   effect(DEF cr, USE op1, USE op2);
15909 
15910   ins_cost(INSN_COST);
15911   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15912 
15913   ins_encode(aarch64_enc_cmpn(op1, op2));
15914 
15915   ins_pipe(icmp_reg_reg);
15916 %}
15917 
15918 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15919 %{
15920   match(Set cr (CmpP op1 zero));
15921 
15922   effect(DEF cr, USE op1, USE zero);
15923 
15924   ins_cost(INSN_COST);
15925   format %{ "cmp  $op1, 0\t // ptr" %}
15926 
15927   ins_encode(aarch64_enc_testp(op1));
15928 
15929   ins_pipe(icmp_reg_imm);
15930 %}
15931 
15932 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15933 %{
15934   match(Set cr (CmpN op1 zero));
15935 
15936   effect(DEF cr, USE op1, USE zero);
15937 
15938   ins_cost(INSN_COST);
15939   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15940 
15941   ins_encode(aarch64_enc_testn(op1));
15942 
15943   ins_pipe(icmp_reg_imm);
15944 %}
15945 
15946 // FP comparisons
15947 //
15948 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15949 // using normal cmpOp. See declaration of rFlagsReg for details.
15950 
15951 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15952 %{
15953   match(Set cr (CmpF src1 src2));
15954 
15955   ins_cost(3 * INSN_COST);
15956   format %{ "fcmps $src1, $src2" %}
15957 
15958   ins_encode %{
15959     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15960   %}
15961 
15962   ins_pipe(pipe_class_compare);
15963 %}
15964 
15965 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15966 %{
15967   match(Set cr (CmpF src1 src2));
15968 
15969   ins_cost(3 * INSN_COST);
15970   format %{ "fcmps $src1, 0.0" %}
15971 
15972   ins_encode %{
15973     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15974   %}
15975 
15976   ins_pipe(pipe_class_compare);
15977 %}
15978 // FROM HERE
15979 
15980 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15981 %{
15982   match(Set cr (CmpD src1 src2));
15983 
15984   ins_cost(3 * INSN_COST);
15985   format %{ "fcmpd $src1, $src2" %}
15986 
15987   ins_encode %{
15988     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15989   %}
15990 
15991   ins_pipe(pipe_class_compare);
15992 %}
15993 
15994 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15995 %{
15996   match(Set cr (CmpD src1 src2));
15997 
15998   ins_cost(3 * INSN_COST);
15999   format %{ "fcmpd $src1, 0.0" %}
16000 
16001   ins_encode %{
16002     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16003   %}
16004 
16005   ins_pipe(pipe_class_compare);
16006 %}
16007 
16008 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16009 %{
16010   match(Set dst (CmpF3 src1 src2));
16011   effect(KILL cr);
16012 
16013   ins_cost(5 * INSN_COST);
16014   format %{ "fcmps $src1, $src2\n\t"
16015             "csinvw($dst, zr, zr, eq\n\t"
16016             "csnegw($dst, $dst, $dst, lt)"
16017   %}
16018 
16019   ins_encode %{
16020     Label done;
16021     FloatRegister s1 = as_FloatRegister($src1$$reg);
16022     FloatRegister s2 = as_FloatRegister($src2$$reg);
16023     Register d = as_Register($dst$$reg);
16024     __ fcmps(s1, s2);
16025     // installs 0 if EQ else -1
16026     __ csinvw(d, zr, zr, Assembler::EQ);
16027     // keeps -1 if less or unordered else installs 1
16028     __ csnegw(d, d, d, Assembler::LT);
16029     __ bind(done);
16030   %}
16031 
16032   ins_pipe(pipe_class_default);
16033 
16034 %}
16035 
16036 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16037 %{
16038   match(Set dst (CmpD3 src1 src2));
16039   effect(KILL cr);
16040 
16041   ins_cost(5 * INSN_COST);
16042   format %{ "fcmpd $src1, $src2\n\t"
16043             "csinvw($dst, zr, zr, eq\n\t"
16044             "csnegw($dst, $dst, $dst, lt)"
16045   %}
16046 
16047   ins_encode %{
16048     Label done;
16049     FloatRegister s1 = as_FloatRegister($src1$$reg);
16050     FloatRegister s2 = as_FloatRegister($src2$$reg);
16051     Register d = as_Register($dst$$reg);
16052     __ fcmpd(s1, s2);
16053     // installs 0 if EQ else -1
16054     __ csinvw(d, zr, zr, Assembler::EQ);
16055     // keeps -1 if less or unordered else installs 1
16056     __ csnegw(d, d, d, Assembler::LT);
16057     __ bind(done);
16058   %}
16059   ins_pipe(pipe_class_default);
16060 
16061 %}
16062 
16063 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16064 %{
16065   match(Set dst (CmpF3 src1 zero));
16066   effect(KILL cr);
16067 
16068   ins_cost(5 * INSN_COST);
16069   format %{ "fcmps $src1, 0.0\n\t"
16070             "csinvw($dst, zr, zr, eq\n\t"
16071             "csnegw($dst, $dst, $dst, lt)"
16072   %}
16073 
16074   ins_encode %{
16075     Label done;
16076     FloatRegister s1 = as_FloatRegister($src1$$reg);
16077     Register d = as_Register($dst$$reg);
16078     __ fcmps(s1, 0.0);
16079     // installs 0 if EQ else -1
16080     __ csinvw(d, zr, zr, Assembler::EQ);
16081     // keeps -1 if less or unordered else installs 1
16082     __ csnegw(d, d, d, Assembler::LT);
16083     __ bind(done);
16084   %}
16085 
16086   ins_pipe(pipe_class_default);
16087 
16088 %}
16089 
16090 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16091 %{
16092   match(Set dst (CmpD3 src1 zero));
16093   effect(KILL cr);
16094 
16095   ins_cost(5 * INSN_COST);
16096   format %{ "fcmpd $src1, 0.0\n\t"
16097             "csinvw($dst, zr, zr, eq\n\t"
16098             "csnegw($dst, $dst, $dst, lt)"
16099   %}
16100 
16101   ins_encode %{
16102     Label done;
16103     FloatRegister s1 = as_FloatRegister($src1$$reg);
16104     Register d = as_Register($dst$$reg);
16105     __ fcmpd(s1, 0.0);
16106     // installs 0 if EQ else -1
16107     __ csinvw(d, zr, zr, Assembler::EQ);
16108     // keeps -1 if less or unordered else installs 1
16109     __ csnegw(d, d, d, Assembler::LT);
16110     __ bind(done);
16111   %}
16112   ins_pipe(pipe_class_default);
16113 
16114 %}
16115 
16116 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16117 %{
16118   match(Set dst (CmpLTMask p q));
16119   effect(KILL cr);
16120 
16121   ins_cost(3 * INSN_COST);
16122 
16123   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16124             "csetw $dst, lt\n\t"
16125             "subw $dst, zr, $dst"
16126   %}
16127 
16128   ins_encode %{
16129     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16130     __ csetw(as_Register($dst$$reg), Assembler::LT);
16131     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16132   %}
16133 
16134   ins_pipe(ialu_reg_reg);
16135 %}
16136 
16137 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16138 %{
16139   match(Set dst (CmpLTMask src zero));
16140   effect(KILL cr);
16141 
16142   ins_cost(INSN_COST);
16143 
16144   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16145 
16146   ins_encode %{
16147     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16148   %}
16149 
16150   ins_pipe(ialu_reg_shift);
16151 %}
16152 
16153 // ============================================================================
16154 // Max and Min
16155 
16156 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16157 %{
16158   effect( DEF dst, USE src1, USE src2, USE cr );
16159 
16160   ins_cost(INSN_COST * 2);
16161   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
16162 
16163   ins_encode %{
16164     __ cselw(as_Register($dst$$reg),
16165              as_Register($src1$$reg),
16166              as_Register($src2$$reg),
16167              Assembler::LT);
16168   %}
16169 
16170   ins_pipe(icond_reg_reg);
16171 %}
16172 
16173 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16174 %{
16175   match(Set dst (MinI src1 src2));
16176   ins_cost(INSN_COST * 3);
16177 
16178   expand %{
16179     rFlagsReg cr;
16180     compI_reg_reg(cr, src1, src2);
16181     cmovI_reg_reg_lt(dst, src1, src2, cr);
16182   %}
16183 
16184 %}
16185 // FROM HERE
16186 
16187 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16188 %{
16189   effect( DEF dst, USE src1, USE src2, USE cr );
16190 
16191   ins_cost(INSN_COST * 2);
16192   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
16193 
16194   ins_encode %{
16195     __ cselw(as_Register($dst$$reg),
16196              as_Register($src1$$reg),
16197              as_Register($src2$$reg),
16198              Assembler::GT);
16199   %}
16200 
16201   ins_pipe(icond_reg_reg);
16202 %}
16203 
16204 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16205 %{
16206   match(Set dst (MaxI src1 src2));
16207   ins_cost(INSN_COST * 3);
16208   expand %{
16209     rFlagsReg cr;
16210     compI_reg_reg(cr, src1, src2);
16211     cmovI_reg_reg_gt(dst, src1, src2, cr);
16212   %}
16213 %}
16214 
16215 // ============================================================================
16216 // Branch Instructions
16217 
16218 // Direct Branch.
16219 instruct branch(label lbl)
16220 %{
16221   match(Goto);
16222 
16223   effect(USE lbl);
16224 
16225   ins_cost(BRANCH_COST);
16226   format %{ "b  $lbl" %}
16227 
16228   ins_encode(aarch64_enc_b(lbl));
16229 
16230   ins_pipe(pipe_branch);
16231 %}
16232 
16233 // Conditional Near Branch
16234 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16235 %{
16236   // Same match rule as `branchConFar'.
16237   match(If cmp cr);
16238 
16239   effect(USE lbl);
16240 
16241   ins_cost(BRANCH_COST);
16242   // If set to 1 this indicates that the current instruction is a
16243   // short variant of a long branch. This avoids using this
16244   // instruction in first-pass matching. It will then only be used in
16245   // the `Shorten_branches' pass.
16246   // ins_short_branch(1);
16247   format %{ "b$cmp  $lbl" %}
16248 
16249   ins_encode(aarch64_enc_br_con(cmp, lbl));
16250 
16251   ins_pipe(pipe_branch_cond);
16252 %}
16253 
16254 // Conditional Near Branch Unsigned
16255 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16256 %{
16257   // Same match rule as `branchConFar'.
16258   match(If cmp cr);
16259 
16260   effect(USE lbl);
16261 
16262   ins_cost(BRANCH_COST);
16263   // If set to 1 this indicates that the current instruction is a
16264   // short variant of a long branch. This avoids using this
16265   // instruction in first-pass matching. It will then only be used in
16266   // the `Shorten_branches' pass.
16267   // ins_short_branch(1);
16268   format %{ "b$cmp  $lbl\t# unsigned" %}
16269 
16270   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16271 
16272   ins_pipe(pipe_branch_cond);
16273 %}
16274 
16275 // Make use of CBZ and CBNZ.  These instructions, as well as being
16276 // shorter than (cmp; branch), have the additional benefit of not
16277 // killing the flags.
16278 
16279 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16280   match(If cmp (CmpI op1 op2));
16281   effect(USE labl);
16282 
16283   ins_cost(BRANCH_COST);
16284   format %{ "cbw$cmp   $op1, $labl" %}
16285   ins_encode %{
16286     Label* L = $labl$$label;
16287     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16288     if (cond == Assembler::EQ)
16289       __ cbzw($op1$$Register, *L);
16290     else
16291       __ cbnzw($op1$$Register, *L);
16292   %}
16293   ins_pipe(pipe_cmp_branch);
16294 %}
16295 
16296 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16297   match(If cmp (CmpL op1 op2));
16298   effect(USE labl);
16299 
16300   ins_cost(BRANCH_COST);
16301   format %{ "cb$cmp   $op1, $labl" %}
16302   ins_encode %{
16303     Label* L = $labl$$label;
16304     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16305     if (cond == Assembler::EQ)
16306       __ cbz($op1$$Register, *L);
16307     else
16308       __ cbnz($op1$$Register, *L);
16309   %}
16310   ins_pipe(pipe_cmp_branch);
16311 %}
16312 
16313 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16314   match(If cmp (CmpP op1 op2));
16315   effect(USE labl);
16316 
16317   ins_cost(BRANCH_COST);
16318   format %{ "cb$cmp   $op1, $labl" %}
16319   ins_encode %{
16320     Label* L = $labl$$label;
16321     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16322     if (cond == Assembler::EQ)
16323       __ cbz($op1$$Register, *L);
16324     else
16325       __ cbnz($op1$$Register, *L);
16326   %}
16327   ins_pipe(pipe_cmp_branch);
16328 %}
16329 
16330 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16331   match(If cmp (CmpN op1 op2));
16332   effect(USE labl);
16333 
16334   ins_cost(BRANCH_COST);
16335   format %{ "cbw$cmp   $op1, $labl" %}
16336   ins_encode %{
16337     Label* L = $labl$$label;
16338     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16339     if (cond == Assembler::EQ)
16340       __ cbzw($op1$$Register, *L);
16341     else
16342       __ cbnzw($op1$$Register, *L);
16343   %}
16344   ins_pipe(pipe_cmp_branch);
16345 %}
16346 
16347 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16348   match(If cmp (CmpP (DecodeN oop) zero));
16349   effect(USE labl);
16350 
16351   ins_cost(BRANCH_COST);
16352   format %{ "cb$cmp   $oop, $labl" %}
16353   ins_encode %{
16354     Label* L = $labl$$label;
16355     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16356     if (cond == Assembler::EQ)
16357       __ cbzw($oop$$Register, *L);
16358     else
16359       __ cbnzw($oop$$Register, *L);
16360   %}
16361   ins_pipe(pipe_cmp_branch);
16362 %}
16363 
16364 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16365   match(If cmp (CmpU op1 op2));
16366   effect(USE labl);
16367 
16368   ins_cost(BRANCH_COST);
16369   format %{ "cbw$cmp   $op1, $labl" %}
16370   ins_encode %{
16371     Label* L = $labl$$label;
16372     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16373     if (cond == Assembler::EQ || cond == Assembler::LS)
16374       __ cbzw($op1$$Register, *L);
16375     else
16376       __ cbnzw($op1$$Register, *L);
16377   %}
16378   ins_pipe(pipe_cmp_branch);
16379 %}
16380 
16381 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16382   match(If cmp (CmpUL op1 op2));
16383   effect(USE labl);
16384 
16385   ins_cost(BRANCH_COST);
16386   format %{ "cb$cmp   $op1, $labl" %}
16387   ins_encode %{
16388     Label* L = $labl$$label;
16389     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16390     if (cond == Assembler::EQ || cond == Assembler::LS)
16391       __ cbz($op1$$Register, *L);
16392     else
16393       __ cbnz($op1$$Register, *L);
16394   %}
16395   ins_pipe(pipe_cmp_branch);
16396 %}
16397 
16398 // Test bit and Branch
16399 
16400 // Patterns for short (< 32KiB) variants
16401 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16402   match(If cmp (CmpL op1 op2));
16403   effect(USE labl);
16404 
16405   ins_cost(BRANCH_COST);
16406   format %{ "cb$cmp   $op1, $labl # long" %}
16407   ins_encode %{
16408     Label* L = $labl$$label;
16409     Assembler::Condition cond =
16410       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16411     __ tbr(cond, $op1$$Register, 63, *L);
16412   %}
16413   ins_pipe(pipe_cmp_branch);
16414   ins_short_branch(1);
16415 %}
16416 
16417 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16418   match(If cmp (CmpI op1 op2));
16419   effect(USE labl);
16420 
16421   ins_cost(BRANCH_COST);
16422   format %{ "cb$cmp   $op1, $labl # int" %}
16423   ins_encode %{
16424     Label* L = $labl$$label;
16425     Assembler::Condition cond =
16426       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16427     __ tbr(cond, $op1$$Register, 31, *L);
16428   %}
16429   ins_pipe(pipe_cmp_branch);
16430   ins_short_branch(1);
16431 %}
16432 
16433 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16434   match(If cmp (CmpL (AndL op1 op2) op3));
16435   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16436   effect(USE labl);
16437 
16438   ins_cost(BRANCH_COST);
16439   format %{ "tb$cmp   $op1, $op2, $labl" %}
16440   ins_encode %{
16441     Label* L = $labl$$label;
16442     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16443     int bit = exact_log2_long($op2$$constant);
16444     __ tbr(cond, $op1$$Register, bit, *L);
16445   %}
16446   ins_pipe(pipe_cmp_branch);
16447   ins_short_branch(1);
16448 %}
16449 
16450 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16451   match(If cmp (CmpI (AndI op1 op2) op3));
16452   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16453   effect(USE labl);
16454 
16455   ins_cost(BRANCH_COST);
16456   format %{ "tb$cmp   $op1, $op2, $labl" %}
16457   ins_encode %{
16458     Label* L = $labl$$label;
16459     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16460     int bit = exact_log2((juint)$op2$$constant);
16461     __ tbr(cond, $op1$$Register, bit, *L);
16462   %}
16463   ins_pipe(pipe_cmp_branch);
16464   ins_short_branch(1);
16465 %}
16466 
16467 // And far variants
16468 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16469   match(If cmp (CmpL op1 op2));
16470   effect(USE labl);
16471 
16472   ins_cost(BRANCH_COST);
16473   format %{ "cb$cmp   $op1, $labl # long" %}
16474   ins_encode %{
16475     Label* L = $labl$$label;
16476     Assembler::Condition cond =
16477       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16478     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16479   %}
16480   ins_pipe(pipe_cmp_branch);
16481 %}
16482 
16483 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16484   match(If cmp (CmpI op1 op2));
16485   effect(USE labl);
16486 
16487   ins_cost(BRANCH_COST);
16488   format %{ "cb$cmp   $op1, $labl # int" %}
16489   ins_encode %{
16490     Label* L = $labl$$label;
16491     Assembler::Condition cond =
16492       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16493     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16494   %}
16495   ins_pipe(pipe_cmp_branch);
16496 %}
16497 
16498 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16499   match(If cmp (CmpL (AndL op1 op2) op3));
16500   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16501   effect(USE labl);
16502 
16503   ins_cost(BRANCH_COST);
16504   format %{ "tb$cmp   $op1, $op2, $labl" %}
16505   ins_encode %{
16506     Label* L = $labl$$label;
16507     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16508     int bit = exact_log2_long($op2$$constant);
16509     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16510   %}
16511   ins_pipe(pipe_cmp_branch);
16512 %}
16513 
16514 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16515   match(If cmp (CmpI (AndI op1 op2) op3));
16516   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16517   effect(USE labl);
16518 
16519   ins_cost(BRANCH_COST);
16520   format %{ "tb$cmp   $op1, $op2, $labl" %}
16521   ins_encode %{
16522     Label* L = $labl$$label;
16523     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16524     int bit = exact_log2((juint)$op2$$constant);
16525     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16526   %}
16527   ins_pipe(pipe_cmp_branch);
16528 %}
16529 
16530 // Test bits
16531 
16532 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16533   match(Set cr (CmpL (AndL op1 op2) op3));
16534   predicate(Assembler::operand_valid_for_logical_immediate
16535             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16536 
16537   ins_cost(INSN_COST);
16538   format %{ "tst $op1, $op2 # long" %}
16539   ins_encode %{
16540     __ tst($op1$$Register, $op2$$constant);
16541   %}
16542   ins_pipe(ialu_reg_reg);
16543 %}
16544 
16545 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16546   match(Set cr (CmpI (AndI op1 op2) op3));
16547   predicate(Assembler::operand_valid_for_logical_immediate
16548             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16549 
16550   ins_cost(INSN_COST);
16551   format %{ "tst $op1, $op2 # int" %}
16552   ins_encode %{
16553     __ tstw($op1$$Register, $op2$$constant);
16554   %}
16555   ins_pipe(ialu_reg_reg);
16556 %}
16557 
16558 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16559   match(Set cr (CmpL (AndL op1 op2) op3));
16560 
16561   ins_cost(INSN_COST);
16562   format %{ "tst $op1, $op2 # long" %}
16563   ins_encode %{
16564     __ tst($op1$$Register, $op2$$Register);
16565   %}
16566   ins_pipe(ialu_reg_reg);
16567 %}
16568 
16569 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16570   match(Set cr (CmpI (AndI op1 op2) op3));
16571 
16572   ins_cost(INSN_COST);
16573   format %{ "tstw $op1, $op2 # int" %}
16574   ins_encode %{
16575     __ tstw($op1$$Register, $op2$$Register);
16576   %}
16577   ins_pipe(ialu_reg_reg);
16578 %}
16579 
16580 
16581 // Conditional Far Branch
16582 // Conditional Far Branch Unsigned
16583 // TODO: fixme
16584 
16585 // counted loop end branch near
16586 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16587 %{
16588   match(CountedLoopEnd cmp cr);
16589 
16590   effect(USE lbl);
16591 
16592   ins_cost(BRANCH_COST);
16593   // short variant.
16594   // ins_short_branch(1);
16595   format %{ "b$cmp $lbl \t// counted loop end" %}
16596 
16597   ins_encode(aarch64_enc_br_con(cmp, lbl));
16598 
16599   ins_pipe(pipe_branch);
16600 %}
16601 
16602 // counted loop end branch near Unsigned
16603 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16604 %{
16605   match(CountedLoopEnd cmp cr);
16606 
16607   effect(USE lbl);
16608 
16609   ins_cost(BRANCH_COST);
16610   // short variant.
16611   // ins_short_branch(1);
16612   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16613 
16614   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16615 
16616   ins_pipe(pipe_branch);
16617 %}
16618 
16619 // counted loop end branch far
16620 // counted loop end branch far unsigned
16621 // TODO: fixme
16622 
16623 // ============================================================================
16624 // inlined locking and unlocking
16625 
16626 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16627 %{
16628   match(Set cr (FastLock object box));
16629   effect(TEMP tmp, TEMP tmp2);
16630 
16631   // TODO
16632   // identify correct cost
16633   ins_cost(5 * INSN_COST);
16634   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16635 
16636   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16637 
16638   ins_pipe(pipe_serial);
16639 %}
16640 
16641 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16642 %{
16643   match(Set cr (FastUnlock object box));
16644   effect(TEMP tmp, TEMP tmp2);
16645 
16646   ins_cost(5 * INSN_COST);
16647   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16648 
16649   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16650 
16651   ins_pipe(pipe_serial);
16652 %}
16653 
16654 
16655 // ============================================================================
16656 // Safepoint Instructions
16657 
16658 // TODO
16659 // provide a near and far version of this code
16660 
16661 instruct safePoint(rFlagsReg cr, iRegP poll)
16662 %{
16663   match(SafePoint poll);
16664   effect(KILL cr);
16665 
16666   format %{
16667     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16668   %}
16669   ins_encode %{
16670     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16671   %}
16672   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16673 %}
16674 
16675 
16676 // ============================================================================
16677 // Procedure Call/Return Instructions
16678 
16679 // Call Java Static Instruction
16680 
16681 instruct CallStaticJavaDirect(method meth)
16682 %{
16683   match(CallStaticJava);
16684 
16685   effect(USE meth);
16686 
16687   ins_cost(CALL_COST);
16688 
16689   format %{ "call,static $meth \t// ==> " %}
16690 
16691   ins_encode(aarch64_enc_java_static_call(meth),
16692              aarch64_enc_call_epilog);
16693 
16694   ins_pipe(pipe_class_call);
16695 %}
16696 
16697 // TO HERE
16698 
16699 // Call Java Dynamic Instruction
16700 instruct CallDynamicJavaDirect(method meth)
16701 %{
16702   match(CallDynamicJava);
16703 
16704   effect(USE meth);
16705 
16706   ins_cost(CALL_COST);
16707 
16708   format %{ "CALL,dynamic $meth \t// ==> " %}
16709 
16710   ins_encode(aarch64_enc_java_dynamic_call(meth),
16711              aarch64_enc_call_epilog);
16712 
16713   ins_pipe(pipe_class_call);
16714 %}
16715 
16716 // Call Runtime Instruction
16717 
16718 instruct CallRuntimeDirect(method meth)
16719 %{
16720   match(CallRuntime);
16721 
16722   effect(USE meth);
16723 
16724   ins_cost(CALL_COST);
16725 
16726   format %{ "CALL, runtime $meth" %}
16727 
16728   ins_encode( aarch64_enc_java_to_runtime(meth) );
16729 
16730   ins_pipe(pipe_class_call);
16731 %}
16732 
16733 // Call Runtime Instruction
16734 
16735 instruct CallLeafDirect(method meth)
16736 %{
16737   match(CallLeaf);
16738 
16739   effect(USE meth);
16740 
16741   ins_cost(CALL_COST);
16742 
16743   format %{ "CALL, runtime leaf $meth" %}
16744 
16745   ins_encode( aarch64_enc_java_to_runtime(meth) );
16746 
16747   ins_pipe(pipe_class_call);
16748 %}
16749 
16750 // Call Runtime Instruction
16751 
16752 instruct CallLeafNoFPDirect(method meth)
16753 %{
16754   match(CallLeafNoFP);
16755 
16756   effect(USE meth);
16757 
16758   ins_cost(CALL_COST);
16759 
16760   format %{ "CALL, runtime leaf nofp $meth" %}
16761 
16762   ins_encode( aarch64_enc_java_to_runtime(meth) );
16763 
16764   ins_pipe(pipe_class_call);
16765 %}
16766 
16767 instruct CallNativeDirect(method meth)
16768 %{
16769   match(CallNative);
16770 
16771   effect(USE meth);
16772 
16773   ins_cost(CALL_COST);
16774 
16775   format %{ "CALL, native $meth" %}
16776 
16777   ins_encode( aarch64_enc_java_to_runtime(meth) );
16778 
16779   ins_pipe(pipe_class_call);
16780 %}
16781 
16782 // Tail Call; Jump from runtime stub to Java code.
16783 // Also known as an 'interprocedural jump'.
16784 // Target of jump will eventually return to caller.
16785 // TailJump below removes the return address.
16786 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16787 %{
16788   match(TailCall jump_target method_ptr);
16789 
16790   ins_cost(CALL_COST);
16791 
16792   format %{ "br $jump_target\t# $method_ptr holds method" %}
16793 
16794   ins_encode(aarch64_enc_tail_call(jump_target));
16795 
16796   ins_pipe(pipe_class_call);
16797 %}
16798 
16799 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16800 %{
16801   match(TailJump jump_target ex_oop);
16802 
16803   ins_cost(CALL_COST);
16804 
16805   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16806 
16807   ins_encode(aarch64_enc_tail_jmp(jump_target));
16808 
16809   ins_pipe(pipe_class_call);
16810 %}
16811 
16812 // Create exception oop: created by stack-crawling runtime code.
16813 // Created exception is now available to this handler, and is setup
16814 // just prior to jumping to this handler. No code emitted.
16815 // TODO check
16816 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16817 instruct CreateException(iRegP_R0 ex_oop)
16818 %{
16819   match(Set ex_oop (CreateEx));
16820 
16821   format %{ " -- \t// exception oop; no code emitted" %}
16822 
16823   size(0);
16824 
16825   ins_encode( /*empty*/ );
16826 
16827   ins_pipe(pipe_class_empty);
16828 %}
16829 
16830 // Rethrow exception: The exception oop will come in the first
16831 // argument position. Then JUMP (not call) to the rethrow stub code.
16832 instruct RethrowException() %{
16833   match(Rethrow);
16834   ins_cost(CALL_COST);
16835 
16836   format %{ "b rethrow_stub" %}
16837 
16838   ins_encode( aarch64_enc_rethrow() );
16839 
16840   ins_pipe(pipe_class_call);
16841 %}
16842 
16843 
16844 // Return Instruction
16845 // epilog node loads ret address into lr as part of frame pop
16846 instruct Ret()
16847 %{
16848   match(Return);
16849 
16850   format %{ "ret\t// return register" %}
16851 
16852   ins_encode( aarch64_enc_ret() );
16853 
16854   ins_pipe(pipe_branch);
16855 %}
16856 
16857 // Die now.
16858 instruct ShouldNotReachHere() %{
16859   match(Halt);
16860 
16861   ins_cost(CALL_COST);
16862   format %{ "ShouldNotReachHere" %}
16863 
16864   ins_encode %{
16865     if (is_reachable()) {
16866       __ stop(_halt_reason);
16867     }
16868   %}
16869 
16870   ins_pipe(pipe_class_default);
16871 %}
16872 
16873 // ============================================================================
16874 // Partial Subtype Check
16875 //
16876 // superklass array for an instance of the superklass.  Set a hidden
16877 // internal cache on a hit (cache is checked with exposed code in
16878 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16879 // encoding ALSO sets flags.
16880 
16881 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16882 %{
16883   match(Set result (PartialSubtypeCheck sub super));
16884   effect(KILL cr, KILL temp);
16885 
16886   ins_cost(1100);  // slightly larger than the next version
16887   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16888 
16889   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16890 
16891   opcode(0x1); // Force zero of result reg on hit
16892 
16893   ins_pipe(pipe_class_memory);
16894 %}
16895 
16896 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16897 %{
16898   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16899   effect(KILL temp, KILL result);
16900 
16901   ins_cost(1100);  // slightly larger than the next version
16902   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16903 
16904   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16905 
16906   opcode(0x0); // Don't zero result reg on hit
16907 
16908   ins_pipe(pipe_class_memory);
16909 %}
16910 
16911 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16912                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16913 %{
16914   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16915   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16916   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16917 
16918   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16919   ins_encode %{
16920     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16921     __ string_compare($str1$$Register, $str2$$Register,
16922                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16923                       $tmp1$$Register, $tmp2$$Register,
16924                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16925   %}
16926   ins_pipe(pipe_class_memory);
16927 %}
16928 
16929 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16930                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16931 %{
16932   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16933   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16934   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16935 
16936   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16937   ins_encode %{
16938     __ string_compare($str1$$Register, $str2$$Register,
16939                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16940                       $tmp1$$Register, $tmp2$$Register,
16941                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16942   %}
16943   ins_pipe(pipe_class_memory);
16944 %}
16945 
16946 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16947                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16948                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16949 %{
16950   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16951   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16952   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16953          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16954 
16955   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16956   ins_encode %{
16957     __ string_compare($str1$$Register, $str2$$Register,
16958                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16959                       $tmp1$$Register, $tmp2$$Register,
16960                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16961                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16962   %}
16963   ins_pipe(pipe_class_memory);
16964 %}
16965 
16966 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16967                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16968                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16969 %{
16970   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16971   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16972   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16973          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16974 
16975   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16976   ins_encode %{
16977     __ string_compare($str1$$Register, $str2$$Register,
16978                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16979                       $tmp1$$Register, $tmp2$$Register,
16980                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16981                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16982   %}
16983   ins_pipe(pipe_class_memory);
16984 %}
16985 
16986 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16987        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16988        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16989 %{
16990   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16991   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16992   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16993          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16994   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16995 
16996   ins_encode %{
16997     __ string_indexof($str1$$Register, $str2$$Register,
16998                       $cnt1$$Register, $cnt2$$Register,
16999                       $tmp1$$Register, $tmp2$$Register,
17000                       $tmp3$$Register, $tmp4$$Register,
17001                       $tmp5$$Register, $tmp6$$Register,
17002                       -1, $result$$Register, StrIntrinsicNode::UU);
17003   %}
17004   ins_pipe(pipe_class_memory);
17005 %}
17006 
17007 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17008        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17009        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17010 %{
17011   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17012   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17013   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17014          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17015   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
17016 
17017   ins_encode %{
17018     __ string_indexof($str1$$Register, $str2$$Register,
17019                       $cnt1$$Register, $cnt2$$Register,
17020                       $tmp1$$Register, $tmp2$$Register,
17021                       $tmp3$$Register, $tmp4$$Register,
17022                       $tmp5$$Register, $tmp6$$Register,
17023                       -1, $result$$Register, StrIntrinsicNode::LL);
17024   %}
17025   ins_pipe(pipe_class_memory);
17026 %}
17027 
17028 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17029        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17030        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17031 %{
17032   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17033   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17034   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17035          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17036   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17037 
17038   ins_encode %{
17039     __ string_indexof($str1$$Register, $str2$$Register,
17040                       $cnt1$$Register, $cnt2$$Register,
17041                       $tmp1$$Register, $tmp2$$Register,
17042                       $tmp3$$Register, $tmp4$$Register,
17043                       $tmp5$$Register, $tmp6$$Register,
17044                       -1, $result$$Register, StrIntrinsicNode::UL);
17045   %}
17046   ins_pipe(pipe_class_memory);
17047 %}
17048 
17049 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17050                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17051                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17052 %{
17053   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17054   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17055   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17056          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17057   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17058 
17059   ins_encode %{
17060     int icnt2 = (int)$int_cnt2$$constant;
17061     __ string_indexof($str1$$Register, $str2$$Register,
17062                       $cnt1$$Register, zr,
17063                       $tmp1$$Register, $tmp2$$Register,
17064                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17065                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17066   %}
17067   ins_pipe(pipe_class_memory);
17068 %}
17069 
17070 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17071                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17072                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17073 %{
17074   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17075   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17076   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17077          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17078   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17079 
17080   ins_encode %{
17081     int icnt2 = (int)$int_cnt2$$constant;
17082     __ string_indexof($str1$$Register, $str2$$Register,
17083                       $cnt1$$Register, zr,
17084                       $tmp1$$Register, $tmp2$$Register,
17085                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17086                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17087   %}
17088   ins_pipe(pipe_class_memory);
17089 %}
17090 
17091 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17092                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17093                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17094 %{
17095   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17096   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17097   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17098          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17099   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17100 
17101   ins_encode %{
17102     int icnt2 = (int)$int_cnt2$$constant;
17103     __ string_indexof($str1$$Register, $str2$$Register,
17104                       $cnt1$$Register, zr,
17105                       $tmp1$$Register, $tmp2$$Register,
17106                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17107                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17108   %}
17109   ins_pipe(pipe_class_memory);
17110 %}
17111 
17112 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17113                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17114                              iRegINoSp tmp3, rFlagsReg cr)
17115 %{
17116   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17117   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17118   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17119          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17120 
17121   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17122 
17123   ins_encode %{
17124     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17125                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17126                            $tmp3$$Register);
17127   %}
17128   ins_pipe(pipe_class_memory);
17129 %}
17130 
17131 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17132                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17133                               iRegINoSp tmp3, rFlagsReg cr)
17134 %{
17135   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17136   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17137   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17138          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17139 
17140   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17141 
17142   ins_encode %{
17143     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17144                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17145                             $tmp3$$Register);
17146   %}
17147   ins_pipe(pipe_class_memory);
17148 %}
17149 
17150 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17151                         iRegI_R0 result, rFlagsReg cr)
17152 %{
17153   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17154   match(Set result (StrEquals (Binary str1 str2) cnt));
17155   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17156 
17157   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17158   ins_encode %{
17159     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17160     __ string_equals($str1$$Register, $str2$$Register,
17161                      $result$$Register, $cnt$$Register, 1);
17162   %}
17163   ins_pipe(pipe_class_memory);
17164 %}
17165 
17166 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17167                         iRegI_R0 result, rFlagsReg cr)
17168 %{
17169   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17170   match(Set result (StrEquals (Binary str1 str2) cnt));
17171   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17172 
17173   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17174   ins_encode %{
17175     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17176     __ string_equals($str1$$Register, $str2$$Register,
17177                      $result$$Register, $cnt$$Register, 2);
17178   %}
17179   ins_pipe(pipe_class_memory);
17180 %}
17181 
17182 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17183                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17184                        iRegP_R10 tmp, rFlagsReg cr)
17185 %{
17186   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17187   match(Set result (AryEq ary1 ary2));
17188   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17189 
17190   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17191   ins_encode %{
17192     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17193                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17194                                    $result$$Register, $tmp$$Register, 1);
17195     if (tpc == NULL) {
17196       ciEnv::current()->record_failure("CodeCache is full");
17197       return;
17198     }
17199   %}
17200   ins_pipe(pipe_class_memory);
17201 %}
17202 
17203 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17204                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17205                        iRegP_R10 tmp, rFlagsReg cr)
17206 %{
17207   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17208   match(Set result (AryEq ary1 ary2));
17209   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17210 
17211   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17212   ins_encode %{
17213     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17214                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17215                                    $result$$Register, $tmp$$Register, 2);
17216     if (tpc == NULL) {
17217       ciEnv::current()->record_failure("CodeCache is full");
17218       return;
17219     }
17220   %}
17221   ins_pipe(pipe_class_memory);
17222 %}
17223 
17224 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17225 %{
17226   match(Set result (CountPositives ary1 len));
17227   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17228   format %{ "count positives byte[] $ary1,$len -> $result" %}
17229   ins_encode %{
17230     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17231     if (tpc == NULL) {
17232       ciEnv::current()->record_failure("CodeCache is full");
17233       return;
17234     }
17235   %}
17236   ins_pipe( pipe_slow );
17237 %}
17238 
17239 // fast char[] to byte[] compression
17240 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17241                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17242                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17243                          iRegI_R0 result, rFlagsReg cr)
17244 %{
17245   match(Set result (StrCompressedCopy src (Binary dst len)));
17246   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17247          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17248 
17249   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
17250   ins_encode %{
17251     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17252                            $result$$Register,
17253                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17254                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17255   %}
17256   ins_pipe(pipe_slow);
17257 %}
17258 
17259 // fast byte[] to char[] inflation
17260 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17261                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17262 %{
17263   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17264   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17265 
17266   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17267   ins_encode %{
17268     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17269                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17270                                         $tmp3$$FloatRegister, $tmp4$$Register);
17271     if (tpc == NULL) {
17272       ciEnv::current()->record_failure("CodeCache is full");
17273       return;
17274     }
17275   %}
17276   ins_pipe(pipe_class_memory);
17277 %}
17278 
17279 // encode char[] to byte[] in ISO_8859_1
17280 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17281                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17282                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17283                           iRegI_R0 result, rFlagsReg cr)
17284 %{
17285   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17286   match(Set result (EncodeISOArray src (Binary dst len)));
17287   effect(USE_KILL src, USE_KILL dst, USE len,
17288          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17289 
17290   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17291   ins_encode %{
17292     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17293                         $result$$Register, false,
17294                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17295                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17296   %}
17297   ins_pipe(pipe_class_memory);
17298 %}
17299 
17300 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17301                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17302                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17303                             iRegI_R0 result, rFlagsReg cr)
17304 %{
17305   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17306   match(Set result (EncodeISOArray src (Binary dst len)));
17307   effect(USE_KILL src, USE_KILL dst, USE len,
17308          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17309 
17310   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17311   ins_encode %{
17312     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17313                         $result$$Register, true,
17314                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17315                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17316   %}
17317   ins_pipe(pipe_class_memory);
17318 %}
17319 
17320 // ============================================================================
17321 // This name is KNOWN by the ADLC and cannot be changed.
17322 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17323 // for this guy.
17324 instruct tlsLoadP(thread_RegP dst)
17325 %{
17326   match(Set dst (ThreadLocal));
17327 
17328   ins_cost(0);
17329 
17330   format %{ " -- \t// $dst=Thread::current(), empty" %}
17331 
17332   size(0);
17333 
17334   ins_encode( /*empty*/ );
17335 
17336   ins_pipe(pipe_class_empty);
17337 %}
17338 
17339 //----------PEEPHOLE RULES-----------------------------------------------------
17340 // These must follow all instruction definitions as they use the names
17341 // defined in the instructions definitions.
17342 //
17343 // peepmatch ( root_instr_name [preceding_instruction]* );
17344 //
17345 // peepconstraint %{
17346 // (instruction_number.operand_name relational_op instruction_number.operand_name
17347 //  [, ...] );
17348 // // instruction numbers are zero-based using left to right order in peepmatch
17349 //
17350 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17351 // // provide an instruction_number.operand_name for each operand that appears
17352 // // in the replacement instruction's match rule
17353 //
17354 // ---------VM FLAGS---------------------------------------------------------
17355 //
17356 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17357 //
17358 // Each peephole rule is given an identifying number starting with zero and
17359 // increasing by one in the order seen by the parser.  An individual peephole
17360 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17361 // on the command-line.
17362 //
17363 // ---------CURRENT LIMITATIONS----------------------------------------------
17364 //
17365 // Only match adjacent instructions in same basic block
17366 // Only equality constraints
17367 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17368 // Only one replacement instruction
17369 //
17370 // ---------EXAMPLE----------------------------------------------------------
17371 //
17372 // // pertinent parts of existing instructions in architecture description
17373 // instruct movI(iRegINoSp dst, iRegI src)
17374 // %{
17375 //   match(Set dst (CopyI src));
17376 // %}
17377 //
17378 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17379 // %{
17380 //   match(Set dst (AddI dst src));
17381 //   effect(KILL cr);
17382 // %}
17383 //
17384 // // Change (inc mov) to lea
17385 // peephole %{
17386 //   // increment preceded by register-register move
17387 //   peepmatch ( incI_iReg movI );
17388 //   // require that the destination register of the increment
17389 //   // match the destination register of the move
17390 //   peepconstraint ( 0.dst == 1.dst );
17391 //   // construct a replacement instruction that sets
17392 //   // the destination to ( move's source register + one )
17393 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17394 // %}
17395 //
17396 
17397 // Implementation no longer uses movX instructions since
17398 // machine-independent system no longer uses CopyX nodes.
17399 //
17400 // peephole
17401 // %{
17402 //   peepmatch (incI_iReg movI);
17403 //   peepconstraint (0.dst == 1.dst);
17404 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17405 // %}
17406 
17407 // peephole
17408 // %{
17409 //   peepmatch (decI_iReg movI);
17410 //   peepconstraint (0.dst == 1.dst);
17411 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17412 // %}
17413 
17414 // peephole
17415 // %{
17416 //   peepmatch (addI_iReg_imm movI);
17417 //   peepconstraint (0.dst == 1.dst);
17418 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17419 // %}
17420 
17421 // peephole
17422 // %{
17423 //   peepmatch (incL_iReg movL);
17424 //   peepconstraint (0.dst == 1.dst);
17425 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17426 // %}
17427 
17428 // peephole
17429 // %{
17430 //   peepmatch (decL_iReg movL);
17431 //   peepconstraint (0.dst == 1.dst);
17432 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17433 // %}
17434 
17435 // peephole
17436 // %{
17437 //   peepmatch (addL_iReg_imm movL);
17438 //   peepconstraint (0.dst == 1.dst);
17439 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17440 // %}
17441 
17442 // peephole
17443 // %{
17444 //   peepmatch (addP_iReg_imm movP);
17445 //   peepconstraint (0.dst == 1.dst);
17446 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17447 // %}
17448 
17449 // // Change load of spilled value to only a spill
17450 // instruct storeI(memory mem, iRegI src)
17451 // %{
17452 //   match(Set mem (StoreI mem src));
17453 // %}
17454 //
17455 // instruct loadI(iRegINoSp dst, memory mem)
17456 // %{
17457 //   match(Set dst (LoadI mem));
17458 // %}
17459 //
17460 
17461 //----------SMARTSPILL RULES---------------------------------------------------
17462 // These must follow all instruction definitions as they use the names
17463 // defined in the instructions definitions.
17464 
17465 // Local Variables:
17466 // mode: c++
17467 // End: