1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly accessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_codestub_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298   bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // Assert that the given node is not a variable shift.
 1315   bool assert_not_var_shift(const Node* n);
 1316 
 1317   // predicate controlling addressing modes
 1318   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1319 %}
 1320 
 1321 source %{
 1322 
 1323   // Derived RegMask with conditionally allocatable registers
 1324 
 1325   void PhaseOutput::pd_perform_mach_node_analysis() {
 1326   }
 1327 
 1328   int MachNode::pd_alignment_required() const {
 1329     return 1;
 1330   }
 1331 
 1332   int MachNode::compute_padding(int current_offset) const {
 1333     return 0;
 1334   }
 1335 
 1336   RegMask _ANY_REG32_mask;
 1337   RegMask _ANY_REG_mask;
 1338   RegMask _PTR_REG_mask;
 1339   RegMask _NO_SPECIAL_REG32_mask;
 1340   RegMask _NO_SPECIAL_REG_mask;
 1341   RegMask _NO_SPECIAL_PTR_REG_mask;
 1342 
 1343   void reg_mask_init() {
 1344     // We derive below RegMask(s) from the ones which are auto-generated from
 1345     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1346     // registers conditionally reserved.
 1347 
 1348     _ANY_REG32_mask = _ALL_REG32_mask;
 1349     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1350 
 1351     _ANY_REG_mask = _ALL_REG_mask;
 1352 
 1353     _PTR_REG_mask = _ALL_REG_mask;
 1354 
 1355     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1356     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1357 
 1358     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1362     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1363 
 1364     // r27 is not allocatable when compressed oops is on and heapbase is not
 1365     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1366     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1367       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1368       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1369       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1370     }
 1371 
 1372     // r29 is not allocatable when PreserveFramePointer is on
 1373     if (PreserveFramePointer) {
 1374       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1375       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1376       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1377     }
 1378   }
 1379 
 1380   // Optimizaton of volatile gets and puts
 1381   // -------------------------------------
 1382   //
 1383   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1384   // use to implement volatile reads and writes. For a volatile read
 1385   // we simply need
 1386   //
 1387   //   ldar<x>
 1388   //
 1389   // and for a volatile write we need
 1390   //
 1391   //   stlr<x>
 1392   //
 1393   // Alternatively, we can implement them by pairing a normal
 1394   // load/store with a memory barrier. For a volatile read we need
 1395   //
 1396   //   ldr<x>
 1397   //   dmb ishld
 1398   //
 1399   // for a volatile write
 1400   //
 1401   //   dmb ish
 1402   //   str<x>
 1403   //   dmb ish
 1404   //
 1405   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1406   // sequences. These are normally translated to an instruction
 1407   // sequence like the following
 1408   //
 1409   //   dmb      ish
 1410   // retry:
 1411   //   ldxr<x>   rval raddr
 1412   //   cmp       rval rold
 1413   //   b.ne done
 1414   //   stlxr<x>  rval, rnew, rold
 1415   //   cbnz      rval retry
 1416   // done:
 1417   //   cset      r0, eq
 1418   //   dmb ishld
 1419   //
 1420   // Note that the exclusive store is already using an stlxr
 1421   // instruction. That is required to ensure visibility to other
 1422   // threads of the exclusive write (assuming it succeeds) before that
 1423   // of any subsequent writes.
 1424   //
 1425   // The following instruction sequence is an improvement on the above
 1426   //
 1427   // retry:
 1428   //   ldaxr<x>  rval raddr
 1429   //   cmp       rval rold
 1430   //   b.ne done
 1431   //   stlxr<x>  rval, rnew, rold
 1432   //   cbnz      rval retry
 1433   // done:
 1434   //   cset      r0, eq
 1435   //
 1436   // We don't need the leading dmb ish since the stlxr guarantees
 1437   // visibility of prior writes in the case that the swap is
 1438   // successful. Crucially we don't have to worry about the case where
 1439   // the swap is not successful since no valid program should be
 1440   // relying on visibility of prior changes by the attempting thread
 1441   // in the case where the CAS fails.
 1442   //
 1443   // Similarly, we don't need the trailing dmb ishld if we substitute
 1444   // an ldaxr instruction since that will provide all the guarantees we
 1445   // require regarding observation of changes made by other threads
 1446   // before any change to the CAS address observed by the load.
 1447   //
 1448   // In order to generate the desired instruction sequence we need to
 1449   // be able to identify specific 'signature' ideal graph node
 1450   // sequences which i) occur as a translation of a volatile reads or
 1451   // writes or CAS operations and ii) do not occur through any other
 1452   // translation or graph transformation. We can then provide
 1453   // alternative aldc matching rules which translate these node
 1454   // sequences to the desired machine code sequences. Selection of the
 1455   // alternative rules can be implemented by predicates which identify
 1456   // the relevant node sequences.
 1457   //
 1458   // The ideal graph generator translates a volatile read to the node
 1459   // sequence
 1460   //
 1461   //   LoadX[mo_acquire]
 1462   //   MemBarAcquire
 1463   //
 1464   // As a special case when using the compressed oops optimization we
 1465   // may also see this variant
 1466   //
 1467   //   LoadN[mo_acquire]
 1468   //   DecodeN
 1469   //   MemBarAcquire
 1470   //
 1471   // A volatile write is translated to the node sequence
 1472   //
 1473   //   MemBarRelease
 1474   //   StoreX[mo_release] {CardMark}-optional
 1475   //   MemBarVolatile
 1476   //
 1477   // n.b. the above node patterns are generated with a strict
 1478   // 'signature' configuration of input and output dependencies (see
 1479   // the predicates below for exact details). The card mark may be as
 1480   // simple as a few extra nodes or, in a few GC configurations, may
 1481   // include more complex control flow between the leading and
 1482   // trailing memory barriers. However, whatever the card mark
 1483   // configuration these signatures are unique to translated volatile
 1484   // reads/stores -- they will not appear as a result of any other
 1485   // bytecode translation or inlining nor as a consequence of
 1486   // optimizing transforms.
 1487   //
 1488   // We also want to catch inlined unsafe volatile gets and puts and
 1489   // be able to implement them using either ldar<x>/stlr<x> or some
 1490   // combination of ldr<x>/stlr<x> and dmb instructions.
 1491   //
 1492   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1493   // normal volatile put node sequence containing an extra cpuorder
 1494   // membar
 1495   //
 1496   //   MemBarRelease
 1497   //   MemBarCPUOrder
 1498   //   StoreX[mo_release] {CardMark}-optional
 1499   //   MemBarCPUOrder
 1500   //   MemBarVolatile
 1501   //
 1502   // n.b. as an aside, a cpuorder membar is not itself subject to
 1503   // matching and translation by adlc rules.  However, the rule
 1504   // predicates need to detect its presence in order to correctly
 1505   // select the desired adlc rules.
 1506   //
 1507   // Inlined unsafe volatile gets manifest as a slightly different
 1508   // node sequence to a normal volatile get because of the
 1509   // introduction of some CPUOrder memory barriers to bracket the
 1510   // Load. However, but the same basic skeleton of a LoadX feeding a
 1511   // MemBarAcquire, possibly through an optional DecodeN, is still
 1512   // present
 1513   //
 1514   //   MemBarCPUOrder
 1515   //        ||       \\
 1516   //   MemBarCPUOrder LoadX[mo_acquire]
 1517   //        ||            |
 1518   //        ||       {DecodeN} optional
 1519   //        ||       /
 1520   //     MemBarAcquire
 1521   //
 1522   // In this case the acquire membar does not directly depend on the
 1523   // load. However, we can be sure that the load is generated from an
 1524   // inlined unsafe volatile get if we see it dependent on this unique
 1525   // sequence of membar nodes. Similarly, given an acquire membar we
 1526   // can know that it was added because of an inlined unsafe volatile
 1527   // get if it is fed and feeds a cpuorder membar and if its feed
 1528   // membar also feeds an acquiring load.
 1529   //
 1530   // Finally an inlined (Unsafe) CAS operation is translated to the
 1531   // following ideal graph
 1532   //
 1533   //   MemBarRelease
 1534   //   MemBarCPUOrder
 1535   //   CompareAndSwapX {CardMark}-optional
 1536   //   MemBarCPUOrder
 1537   //   MemBarAcquire
 1538   //
 1539   // So, where we can identify these volatile read and write
 1540   // signatures we can choose to plant either of the above two code
 1541   // sequences. For a volatile read we can simply plant a normal
 1542   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1543   // also choose to inhibit translation of the MemBarAcquire and
 1544   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1545   //
 1546   // When we recognise a volatile store signature we can choose to
 1547   // plant at a dmb ish as a translation for the MemBarRelease, a
 1548   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1549   // Alternatively, we can inhibit translation of the MemBarRelease
 1550   // and MemBarVolatile and instead plant a simple stlr<x>
 1551   // instruction.
 1552   //
 1553   // when we recognise a CAS signature we can choose to plant a dmb
 1554   // ish as a translation for the MemBarRelease, the conventional
 1555   // macro-instruction sequence for the CompareAndSwap node (which
 1556   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1557   // Alternatively, we can elide generation of the dmb instructions
 1558   // and plant the alternative CompareAndSwap macro-instruction
 1559   // sequence (which uses ldaxr<x>).
 1560   //
 1561   // Of course, the above only applies when we see these signature
 1562   // configurations. We still want to plant dmb instructions in any
 1563   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1564   // MemBarVolatile. For example, at the end of a constructor which
 1565   // writes final/volatile fields we will see a MemBarRelease
 1566   // instruction and this needs a 'dmb ish' lest we risk the
 1567   // constructed object being visible without making the
 1568   // final/volatile field writes visible.
 1569   //
 1570   // n.b. the translation rules below which rely on detection of the
 1571   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1572   // If we see anything other than the signature configurations we
 1573   // always just translate the loads and stores to ldr<x> and str<x>
 1574   // and translate acquire, release and volatile membars to the
 1575   // relevant dmb instructions.
 1576   //
 1577 
 1578   // is_CAS(int opcode, bool maybe_volatile)
 1579   //
 1580   // return true if opcode is one of the possible CompareAndSwapX
 1581   // values otherwise false.
 1582 
 1583   bool is_CAS(int opcode, bool maybe_volatile)
 1584   {
 1585     switch(opcode) {
 1586       // We handle these
 1587     case Op_CompareAndSwapI:
 1588     case Op_CompareAndSwapL:
 1589     case Op_CompareAndSwapP:
 1590     case Op_CompareAndSwapN:
 1591     case Op_ShenandoahCompareAndSwapP:
 1592     case Op_ShenandoahCompareAndSwapN:
 1593     case Op_CompareAndSwapB:
 1594     case Op_CompareAndSwapS:
 1595     case Op_GetAndSetI:
 1596     case Op_GetAndSetL:
 1597     case Op_GetAndSetP:
 1598     case Op_GetAndSetN:
 1599     case Op_GetAndAddI:
 1600     case Op_GetAndAddL:
 1601       return true;
 1602     case Op_CompareAndExchangeI:
 1603     case Op_CompareAndExchangeN:
 1604     case Op_CompareAndExchangeB:
 1605     case Op_CompareAndExchangeS:
 1606     case Op_CompareAndExchangeL:
 1607     case Op_CompareAndExchangeP:
 1608     case Op_WeakCompareAndSwapB:
 1609     case Op_WeakCompareAndSwapS:
 1610     case Op_WeakCompareAndSwapI:
 1611     case Op_WeakCompareAndSwapL:
 1612     case Op_WeakCompareAndSwapP:
 1613     case Op_WeakCompareAndSwapN:
 1614     case Op_ShenandoahWeakCompareAndSwapP:
 1615     case Op_ShenandoahWeakCompareAndSwapN:
 1616     case Op_ShenandoahCompareAndExchangeP:
 1617     case Op_ShenandoahCompareAndExchangeN:
 1618       return maybe_volatile;
 1619     default:
 1620       return false;
 1621     }
 1622   }
 1623 
 1624   // helper to determine the maximum number of Phi nodes we may need to
 1625   // traverse when searching from a card mark membar for the merge mem
 1626   // feeding a trailing membar or vice versa
 1627 
 1628 // predicates controlling emit of ldr<x>/ldar<x>
 1629 
 1630 bool unnecessary_acquire(const Node *barrier)
 1631 {
 1632   assert(barrier->is_MemBar(), "expecting a membar");
 1633 
 1634   MemBarNode* mb = barrier->as_MemBar();
 1635 
 1636   if (mb->trailing_load()) {
 1637     return true;
 1638   }
 1639 
 1640   if (mb->trailing_load_store()) {
 1641     Node* load_store = mb->in(MemBarNode::Precedent);
 1642     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1643     return is_CAS(load_store->Opcode(), true);
 1644   }
 1645 
 1646   return false;
 1647 }
 1648 
 1649 bool needs_acquiring_load(const Node *n)
 1650 {
 1651   assert(n->is_Load(), "expecting a load");
 1652   LoadNode *ld = n->as_Load();
 1653   return ld->is_acquire();
 1654 }
 1655 
 1656 bool unnecessary_release(const Node *n)
 1657 {
 1658   assert((n->is_MemBar() &&
 1659           n->Opcode() == Op_MemBarRelease),
 1660          "expecting a release membar");
 1661 
 1662   MemBarNode *barrier = n->as_MemBar();
 1663   if (!barrier->leading()) {
 1664     return false;
 1665   } else {
 1666     Node* trailing = barrier->trailing_membar();
 1667     MemBarNode* trailing_mb = trailing->as_MemBar();
 1668     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1669     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1670 
 1671     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1672     if (mem->is_Store()) {
 1673       assert(mem->as_Store()->is_release(), "");
 1674       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1675       return true;
 1676     } else {
 1677       assert(mem->is_LoadStore(), "");
 1678       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1679       return is_CAS(mem->Opcode(), true);
 1680     }
 1681   }
 1682   return false;
 1683 }
 1684 
 1685 bool unnecessary_volatile(const Node *n)
 1686 {
 1687   // assert n->is_MemBar();
 1688   MemBarNode *mbvol = n->as_MemBar();
 1689 
 1690   bool release = mbvol->trailing_store();
 1691   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1692 #ifdef ASSERT
 1693   if (release) {
 1694     Node* leading = mbvol->leading_membar();
 1695     assert(leading->Opcode() == Op_MemBarRelease, "");
 1696     assert(leading->as_MemBar()->leading_store(), "");
 1697     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1698   }
 1699 #endif
 1700 
 1701   return release;
 1702 }
 1703 
 1704 // predicates controlling emit of str<x>/stlr<x>
 1705 
 1706 bool needs_releasing_store(const Node *n)
 1707 {
 1708   // assert n->is_Store();
 1709   StoreNode *st = n->as_Store();
 1710   return st->trailing_membar() != NULL;
 1711 }
 1712 
 1713 // predicate controlling translation of CAS
 1714 //
 1715 // returns true if CAS needs to use an acquiring load otherwise false
 1716 
 1717 bool needs_acquiring_load_exclusive(const Node *n)
 1718 {
 1719   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1720   LoadStoreNode* ldst = n->as_LoadStore();
 1721   if (is_CAS(n->Opcode(), false)) {
 1722     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1723   } else {
 1724     return ldst->trailing_membar() != NULL;
 1725   }
 1726 
 1727   // so we can just return true here
 1728   return true;
 1729 }
 1730 
 1731 // Assert that the given node is not a variable shift.
 1732 bool assert_not_var_shift(const Node* n) {
 1733   assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
 1734   return true;
 1735 }
 1736 
 1737 #define __ _masm.
 1738 
 1739 // advance declarations for helper functions to convert register
 1740 // indices to register objects
 1741 
 1742 // the ad file has to provide implementations of certain methods
 1743 // expected by the generic code
 1744 //
 1745 // REQUIRED FUNCTIONALITY
 1746 
 1747 //=============================================================================
 1748 
 1749 // !!!!! Special hack to get all types of calls to specify the byte offset
 1750 //       from the start of the call to the point where the return address
 1751 //       will point.
 1752 
 1753 int MachCallStaticJavaNode::ret_addr_offset()
 1754 {
 1755   // call should be a simple bl
 1756   int off = 4;
 1757   return off;
 1758 }
 1759 
 1760 int MachCallDynamicJavaNode::ret_addr_offset()
 1761 {
 1762   return 16; // movz, movk, movk, bl
 1763 }
 1764 
 1765 int MachCallRuntimeNode::ret_addr_offset() {
 1766   // for generated stubs the call will be
 1767   //   bl(addr)
 1768   // or with far branches
 1769   //   bl(trampoline_stub)
 1770   // for real runtime callouts it will be six instructions
 1771   // see aarch64_enc_java_to_runtime
 1772   //   adr(rscratch2, retaddr)
 1773   //   lea(rscratch1, RuntimeAddress(addr)
 1774   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1775   //   blr(rscratch1)
 1776   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1777   if (cb) {
 1778     return 1 * NativeInstruction::instruction_size;
 1779   } else {
 1780     return 6 * NativeInstruction::instruction_size;
 1781   }
 1782 }
 1783 
 1784 int MachCallNativeNode::ret_addr_offset() {
 1785   // This is implemented using aarch64_enc_java_to_runtime as above.
 1786   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1787   if (cb) {
 1788     return 1 * NativeInstruction::instruction_size;
 1789   } else {
 1790     return 6 * NativeInstruction::instruction_size;
 1791   }
 1792 }
 1793 
 1794 //=============================================================================
 1795 
 1796 #ifndef PRODUCT
 1797 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1798   st->print("BREAKPOINT");
 1799 }
 1800 #endif
 1801 
 1802 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1803   C2_MacroAssembler _masm(&cbuf);
 1804   __ brk(0);
 1805 }
 1806 
 1807 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1808   return MachNode::size(ra_);
 1809 }
 1810 
 1811 //=============================================================================
 1812 
 1813 #ifndef PRODUCT
 1814   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1815     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1816   }
 1817 #endif
 1818 
 1819   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1820     C2_MacroAssembler _masm(&cbuf);
 1821     for (int i = 0; i < _count; i++) {
 1822       __ nop();
 1823     }
 1824   }
 1825 
 1826   uint MachNopNode::size(PhaseRegAlloc*) const {
 1827     return _count * NativeInstruction::instruction_size;
 1828   }
 1829 
 1830 //=============================================================================
 1831 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1832 
 1833 int ConstantTable::calculate_table_base_offset() const {
 1834   return 0;  // absolute addressing, no offset
 1835 }
 1836 
 1837 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1838 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1839   ShouldNotReachHere();
 1840 }
 1841 
 1842 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1843   // Empty encoding
 1844 }
 1845 
 1846 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1847   return 0;
 1848 }
 1849 
 1850 #ifndef PRODUCT
 1851 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1852   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1853 }
 1854 #endif
 1855 
 1856 #ifndef PRODUCT
 1857 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1858   Compile* C = ra_->C;
 1859 
 1860   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1861 
 1862   if (C->output()->need_stack_bang(framesize))
 1863     st->print("# stack bang size=%d\n\t", framesize);
 1864 
 1865   if (VM_Version::use_rop_protection()) {
 1866     st->print("ldr zr, [lr]\n\t");
 1867     st->print("pacia  lr, rfp\n\t");
 1868   }
 1869   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1870     st->print("sub  sp, sp, #%d\n\t", framesize);
 1871     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1872     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1873   } else {
 1874     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1875     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1876     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1877     st->print("sub  sp, sp, rscratch1");
 1878   }
 1879   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1880     st->print("\n\t");
 1881     st->print("ldr  rscratch1, [guard]\n\t");
 1882     st->print("dmb ishld\n\t");
 1883     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1884     st->print("cmp  rscratch1, rscratch2\n\t");
 1885     st->print("b.eq skip");
 1886     st->print("\n\t");
 1887     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1888     st->print("b skip\n\t");
 1889     st->print("guard: int\n\t");
 1890     st->print("\n\t");
 1891     st->print("skip:\n\t");
 1892   }
 1893 }
 1894 #endif
 1895 
 1896 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1897   Compile* C = ra_->C;
 1898   C2_MacroAssembler _masm(&cbuf);
 1899 
 1900   // n.b. frame size includes space for return pc and rfp
 1901   const int framesize = C->output()->frame_size_in_bytes();
 1902 
 1903   // insert a nop at the start of the prolog so we can patch in a
 1904   // branch if we need to invalidate the method later
 1905   __ nop();
 1906 
 1907   if (C->clinit_barrier_on_entry()) {
 1908     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1909 
 1910     Label L_skip_barrier;
 1911 
 1912     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1913     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1914     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1915     __ bind(L_skip_barrier);
 1916   }
 1917 
 1918   if (C->max_vector_size() > 0) {
 1919     __ reinitialize_ptrue();
 1920   }
 1921 
 1922   int bangsize = C->output()->bang_size_in_bytes();
 1923   if (C->output()->need_stack_bang(bangsize))
 1924     __ generate_stack_overflow_check(bangsize);
 1925 
 1926   __ build_frame(framesize);
 1927 
 1928   if (C->stub_function() == NULL) {
 1929     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1930     bs->nmethod_entry_barrier(&_masm);
 1931   }
 1932 
 1933   if (VerifyStackAtCalls) {
 1934     Unimplemented();
 1935   }
 1936 
 1937   C->output()->set_frame_complete(cbuf.insts_size());
 1938 
 1939   if (C->has_mach_constant_base_node()) {
 1940     // NOTE: We set the table base offset here because users might be
 1941     // emitted before MachConstantBaseNode.
 1942     ConstantTable& constant_table = C->output()->constant_table();
 1943     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1944   }
 1945 }
 1946 
 1947 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1948 {
 1949   return MachNode::size(ra_); // too many variables; just compute it
 1950                               // the hard way
 1951 }
 1952 
 1953 int MachPrologNode::reloc() const
 1954 {
 1955   return 0;
 1956 }
 1957 
 1958 //=============================================================================
 1959 
 1960 #ifndef PRODUCT
 1961 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1962   Compile* C = ra_->C;
 1963   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1964 
 1965   st->print("# pop frame %d\n\t",framesize);
 1966 
 1967   if (framesize == 0) {
 1968     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1969   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1970     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1971     st->print("add  sp, sp, #%d\n\t", framesize);
 1972   } else {
 1973     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1974     st->print("add  sp, sp, rscratch1\n\t");
 1975     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1976   }
 1977   if (VM_Version::use_rop_protection()) {
 1978     st->print("autia lr, rfp\n\t");
 1979     st->print("ldr zr, [lr]\n\t");
 1980   }
 1981 
 1982   if (do_polling() && C->is_method_compilation()) {
 1983     st->print("# test polling word\n\t");
 1984     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1985     st->print("cmp  sp, rscratch1\n\t");
 1986     st->print("bhi #slow_path");
 1987   }
 1988 }
 1989 #endif
 1990 
 1991 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1992   Compile* C = ra_->C;
 1993   C2_MacroAssembler _masm(&cbuf);
 1994   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1995 
 1996   __ remove_frame(framesize);
 1997 
 1998   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1999     __ reserved_stack_check();
 2000   }
 2001 
 2002   if (do_polling() && C->is_method_compilation()) {
 2003     Label dummy_label;
 2004     Label* code_stub = &dummy_label;
 2005     if (!C->output()->in_scratch_emit_size()) {
 2006       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 2007     }
 2008     __ relocate(relocInfo::poll_return_type);
 2009     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 2010   }
 2011 }
 2012 
 2013 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 2014   // Variable size. Determine dynamically.
 2015   return MachNode::size(ra_);
 2016 }
 2017 
 2018 int MachEpilogNode::reloc() const {
 2019   // Return number of relocatable values contained in this instruction.
 2020   return 1; // 1 for polling page.
 2021 }
 2022 
 2023 const Pipeline * MachEpilogNode::pipeline() const {
 2024   return MachNode::pipeline_class();
 2025 }
 2026 
 2027 //=============================================================================
 2028 
 2029 // Figure out which register class each belongs in: rc_int, rc_float or
 2030 // rc_stack.
 2031 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2032 
 2033 static enum RC rc_class(OptoReg::Name reg) {
 2034 
 2035   if (reg == OptoReg::Bad) {
 2036     return rc_bad;
 2037   }
 2038 
 2039   // we have 32 int registers * 2 halves
 2040   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2041 
 2042   if (reg < slots_of_int_registers) {
 2043     return rc_int;
 2044   }
 2045 
 2046   // we have 32 float register * 8 halves
 2047   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2048   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2049     return rc_float;
 2050   }
 2051 
 2052   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2053   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2054     return rc_predicate;
 2055   }
 2056 
 2057   // Between predicate regs & stack is the flags.
 2058   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2059 
 2060   return rc_stack;
 2061 }
 2062 
 2063 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2064   Compile* C = ra_->C;
 2065 
 2066   // Get registers to move.
 2067   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2068   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2069   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2070   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2071 
 2072   enum RC src_hi_rc = rc_class(src_hi);
 2073   enum RC src_lo_rc = rc_class(src_lo);
 2074   enum RC dst_hi_rc = rc_class(dst_hi);
 2075   enum RC dst_lo_rc = rc_class(dst_lo);
 2076 
 2077   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2078 
 2079   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 2080     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2081            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2082            "expected aligned-adjacent pairs");
 2083   }
 2084 
 2085   if (src_lo == dst_lo && src_hi == dst_hi) {
 2086     return 0;            // Self copy, no move.
 2087   }
 2088 
 2089   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2090               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2091   int src_offset = ra_->reg2offset(src_lo);
 2092   int dst_offset = ra_->reg2offset(dst_lo);
 2093 
 2094   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2095     uint ireg = ideal_reg();
 2096     if (ireg == Op_VecA && cbuf) {
 2097       C2_MacroAssembler _masm(cbuf);
 2098       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2099       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2100         // stack->stack
 2101         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2102                                                 sve_vector_reg_size_in_bytes);
 2103       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2104         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2105                             sve_vector_reg_size_in_bytes);
 2106       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2107         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2108                               sve_vector_reg_size_in_bytes);
 2109       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2110         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2111                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2112                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2113       } else {
 2114         ShouldNotReachHere();
 2115       }
 2116     } else if (cbuf) {
 2117       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2118       C2_MacroAssembler _masm(cbuf);
 2119       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2120       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2121         // stack->stack
 2122         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2123         if (ireg == Op_VecD) {
 2124           __ unspill(rscratch1, true, src_offset);
 2125           __ spill(rscratch1, true, dst_offset);
 2126         } else {
 2127           __ spill_copy128(src_offset, dst_offset);
 2128         }
 2129       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2130         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2131                ireg == Op_VecD ? __ T8B : __ T16B,
 2132                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2133       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2134         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2135                  ireg == Op_VecD ? __ D : __ Q,
 2136                  ra_->reg2offset(dst_lo));
 2137       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2138         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2139                    ireg == Op_VecD ? __ D : __ Q,
 2140                    ra_->reg2offset(src_lo));
 2141       } else {
 2142         ShouldNotReachHere();
 2143       }
 2144     }
 2145   } else if (cbuf) {
 2146     C2_MacroAssembler _masm(cbuf);
 2147     switch (src_lo_rc) {
 2148     case rc_int:
 2149       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2150         if (is64) {
 2151             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2152                    as_Register(Matcher::_regEncode[src_lo]));
 2153         } else {
 2154             C2_MacroAssembler _masm(cbuf);
 2155             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2156                     as_Register(Matcher::_regEncode[src_lo]));
 2157         }
 2158       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2159         if (is64) {
 2160             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2161                      as_Register(Matcher::_regEncode[src_lo]));
 2162         } else {
 2163             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2164                      as_Register(Matcher::_regEncode[src_lo]));
 2165         }
 2166       } else {                    // gpr --> stack spill
 2167         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2168         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2169       }
 2170       break;
 2171     case rc_float:
 2172       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2173         if (is64) {
 2174             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2175                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2176         } else {
 2177             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2178                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2179         }
 2180       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2181         if (is64) {
 2182             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2183                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2184         } else {
 2185             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2186                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2187         }
 2188       } else {                    // fpr --> stack spill
 2189         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2190         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2191                  is64 ? __ D : __ S, dst_offset);
 2192       }
 2193       break;
 2194     case rc_stack:
 2195       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2196         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2197       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2198         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2199                    is64 ? __ D : __ S, src_offset);
 2200       } else if (dst_lo_rc == rc_predicate) {
 2201         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2202                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2203       } else {                    // stack --> stack copy
 2204         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2205         if (ideal_reg() == Op_RegVectMask) {
 2206           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2207                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2208         } else {
 2209           __ unspill(rscratch1, is64, src_offset);
 2210           __ spill(rscratch1, is64, dst_offset);
 2211         }
 2212       }
 2213       break;
 2214     case rc_predicate:
 2215       if (dst_lo_rc == rc_predicate) {
 2216         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2217       } else if (dst_lo_rc == rc_stack) {
 2218         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2219                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2220       } else {
 2221         assert(false, "bad src and dst rc_class combination.");
 2222         ShouldNotReachHere();
 2223       }
 2224       break;
 2225     default:
 2226       assert(false, "bad rc_class for spill");
 2227       ShouldNotReachHere();
 2228     }
 2229   }
 2230 
 2231   if (st) {
 2232     st->print("spill ");
 2233     if (src_lo_rc == rc_stack) {
 2234       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2235     } else {
 2236       st->print("%s -> ", Matcher::regName[src_lo]);
 2237     }
 2238     if (dst_lo_rc == rc_stack) {
 2239       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2240     } else {
 2241       st->print("%s", Matcher::regName[dst_lo]);
 2242     }
 2243     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2244       int vsize = 0;
 2245       switch (ideal_reg()) {
 2246       case Op_VecD:
 2247         vsize = 64;
 2248         break;
 2249       case Op_VecX:
 2250         vsize = 128;
 2251         break;
 2252       case Op_VecA:
 2253         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2254         break;
 2255       default:
 2256         assert(false, "bad register type for spill");
 2257         ShouldNotReachHere();
 2258       }
 2259       st->print("\t# vector spill size = %d", vsize);
 2260     } else if (ideal_reg() == Op_RegVectMask) {
 2261       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2262       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2263       st->print("\t# predicate spill size = %d", vsize);
 2264     } else {
 2265       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2266     }
 2267   }
 2268 
 2269   return 0;
 2270 
 2271 }
 2272 
 2273 #ifndef PRODUCT
 2274 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2275   if (!ra_)
 2276     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2277   else
 2278     implementation(NULL, ra_, false, st);
 2279 }
 2280 #endif
 2281 
 2282 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2283   implementation(&cbuf, ra_, false, NULL);
 2284 }
 2285 
 2286 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2287   return MachNode::size(ra_);
 2288 }
 2289 
 2290 //=============================================================================
 2291 
 2292 #ifndef PRODUCT
 2293 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2294   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2295   int reg = ra_->get_reg_first(this);
 2296   st->print("add %s, rsp, #%d]\t# box lock",
 2297             Matcher::regName[reg], offset);
 2298 }
 2299 #endif
 2300 
 2301 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2302   C2_MacroAssembler _masm(&cbuf);
 2303 
 2304   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2305   int reg    = ra_->get_encode(this);
 2306 
 2307   // This add will handle any 24-bit signed offset. 24 bits allows an
 2308   // 8 megabyte stack frame.
 2309   __ add(as_Register(reg), sp, offset);
 2310 }
 2311 
 2312 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2313   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2314   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2315 
 2316   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2317     return NativeInstruction::instruction_size;
 2318   } else {
 2319     return 2 * NativeInstruction::instruction_size;
 2320   }
 2321 }
 2322 
 2323 //=============================================================================
 2324 
 2325 #ifndef PRODUCT
 2326 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2327 {
 2328   st->print_cr("# MachUEPNode");
 2329   if (UseCompressedClassPointers) {
 2330     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2331     if (CompressedKlassPointers::shift() != 0) {
 2332       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2333     }
 2334   } else {
 2335    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2336   }
 2337   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2338   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2339 }
 2340 #endif
 2341 
 2342 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2343 {
 2344   // This is the unverified entry point.
 2345   C2_MacroAssembler _masm(&cbuf);
 2346 
 2347   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2348   Label skip;
 2349   // TODO
 2350   // can we avoid this skip and still use a reloc?
 2351   __ br(Assembler::EQ, skip);
 2352   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2353   __ bind(skip);
 2354 }
 2355 
 2356 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2357 {
 2358   return MachNode::size(ra_);
 2359 }
 2360 
 2361 // REQUIRED EMIT CODE
 2362 
 2363 //=============================================================================
 2364 
 2365 // Emit exception handler code.
 2366 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2367 {
 2368   // mov rscratch1 #exception_blob_entry_point
 2369   // br rscratch1
 2370   // Note that the code buffer's insts_mark is always relative to insts.
 2371   // That's why we must use the macroassembler to generate a handler.
 2372   C2_MacroAssembler _masm(&cbuf);
 2373   address base = __ start_a_stub(size_exception_handler());
 2374   if (base == NULL) {
 2375     ciEnv::current()->record_failure("CodeCache is full");
 2376     return 0;  // CodeBuffer::expand failed
 2377   }
 2378   int offset = __ offset();
 2379   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2380   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2381   __ end_a_stub();
 2382   return offset;
 2383 }
 2384 
 2385 // Emit deopt handler code.
 2386 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2387 {
 2388   // Note that the code buffer's insts_mark is always relative to insts.
 2389   // That's why we must use the macroassembler to generate a handler.
 2390   C2_MacroAssembler _masm(&cbuf);
 2391   address base = __ start_a_stub(size_deopt_handler());
 2392   if (base == NULL) {
 2393     ciEnv::current()->record_failure("CodeCache is full");
 2394     return 0;  // CodeBuffer::expand failed
 2395   }
 2396   int offset = __ offset();
 2397 
 2398   __ adr(lr, __ pc());
 2399   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2400 
 2401   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2402   __ end_a_stub();
 2403   return offset;
 2404 }
 2405 
 2406 // REQUIRED MATCHER CODE
 2407 
 2408 //=============================================================================
 2409 
 2410 const bool Matcher::match_rule_supported(int opcode) {
 2411   if (!has_match_rule(opcode))
 2412     return false;
 2413 
 2414   bool ret_value = true;
 2415   switch (opcode) {
 2416     case Op_OnSpinWait:
 2417       return VM_Version::supports_on_spin_wait();
 2418     case Op_CacheWB:
 2419     case Op_CacheWBPreSync:
 2420     case Op_CacheWBPostSync:
 2421       if (!VM_Version::supports_data_cache_line_flush()) {
 2422         ret_value = false;
 2423       }
 2424       break;
 2425     case Op_LoadVectorMasked:
 2426     case Op_StoreVectorMasked:
 2427     case Op_LoadVectorGatherMasked:
 2428     case Op_StoreVectorScatterMasked:
 2429     case Op_MaskAll:
 2430     case Op_AndVMask:
 2431     case Op_OrVMask:
 2432     case Op_XorVMask:
 2433       if (UseSVE == 0) {
 2434         ret_value = false;
 2435       }
 2436       break;
 2437   }
 2438 
 2439   return ret_value; // Per default match rules are supported.
 2440 }
 2441 
 2442 // Identify extra cases that we might want to provide match rules for vector nodes and
 2443 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2444 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2445   if (!match_rule_supported(opcode)) {
 2446     return false;
 2447   }
 2448   int bit_size = vlen * type2aelembytes(bt) * 8;
 2449   if (UseSVE == 0 && bit_size > 128) {
 2450     return false;
 2451   }
 2452   if (UseSVE > 0) {
 2453     return op_sve_supported(opcode, vlen, bt);
 2454   } else { // NEON
 2455     // Special cases
 2456     switch (opcode) {
 2457     case Op_VectorMaskCmp:
 2458       if (vlen < 2 || bit_size < 64) {
 2459         return false;
 2460       }
 2461       break;
 2462     case Op_MulAddVS2VI:
 2463       if (bit_size < 128) {
 2464         return false;
 2465       }
 2466       break;
 2467     case Op_MulVL:
 2468       return false;
 2469     case Op_VectorLoadShuffle:
 2470     case Op_VectorRearrange:
 2471       if (vlen < 4) {
 2472         return false;
 2473       }
 2474       break;
 2475     case Op_LoadVectorGather:
 2476     case Op_StoreVectorScatter:
 2477       return false;
 2478     default:
 2479       break;
 2480     }
 2481   }
 2482   return vector_size_supported(bt, vlen);
 2483 }
 2484 
 2485 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
 2486   // Only SVE supports masked operations.
 2487   if (UseSVE == 0) {
 2488     return false;
 2489   }
 2490   return match_rule_supported(opcode) &&
 2491          masked_op_sve_supported(opcode, vlen, bt);
 2492 }
 2493 
 2494 const RegMask* Matcher::predicate_reg_mask(void) {
 2495   return &_PR_REG_mask;
 2496 }
 2497 
 2498 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2499   return new TypeVectMask(elemTy, length);
 2500 }
 2501 
 2502 // Vector calling convention not yet implemented.
 2503 const bool Matcher::supports_vector_calling_convention(void) {
 2504   return false;
 2505 }
 2506 
 2507 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2508   Unimplemented();
 2509   return OptoRegPair(0, 0);
 2510 }
 2511 
 2512 // Is this branch offset short enough that a short branch can be used?
 2513 //
 2514 // NOTE: If the platform does not provide any short branch variants, then
 2515 //       this method should return false for offset 0.
 2516 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2517   // The passed offset is relative to address of the branch.
 2518 
 2519   return (-32768 <= offset && offset < 32768);
 2520 }
 2521 
 2522 // Vector width in bytes.
 2523 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2524   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2525   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2526   // Minimum 2 values in vector
 2527   if (size < 2*type2aelembytes(bt)) size = 0;
 2528   // But never < 4
 2529   if (size < 4) size = 0;
 2530   return size;
 2531 }
 2532 
 2533 // Limits on vector size (number of elements) loaded into vector.
 2534 const int Matcher::max_vector_size(const BasicType bt) {
 2535   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2536 }
 2537 
 2538 const int Matcher::min_vector_size(const BasicType bt) {
 2539   int max_size = max_vector_size(bt);
 2540   // Limit the min vector size to 8 bytes.
 2541   int size = 8 / type2aelembytes(bt);
 2542   if (bt == T_BYTE) {
 2543     // To support vector api shuffle/rearrange.
 2544     size = 4;
 2545   } else if (bt == T_BOOLEAN) {
 2546     // To support vector api load/store mask.
 2547     size = 2;
 2548   }
 2549   if (size < 2) size = 2;
 2550   return MIN2(size, max_size);
 2551 }
 2552 
 2553 // Actual max scalable vector register length.
 2554 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2555   return Matcher::max_vector_size(bt);
 2556 }
 2557 
 2558 // Vector ideal reg.
 2559 const uint Matcher::vector_ideal_reg(int len) {
 2560   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2561     return Op_VecA;
 2562   }
 2563   switch(len) {
 2564     // For 16-bit/32-bit mask vector, reuse VecD.
 2565     case  2:
 2566     case  4:
 2567     case  8: return Op_VecD;
 2568     case 16: return Op_VecX;
 2569   }
 2570   ShouldNotReachHere();
 2571   return 0;
 2572 }
 2573 
 2574 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2575   ShouldNotReachHere(); // generic vector operands not supported
 2576   return NULL;
 2577 }
 2578 
 2579 bool Matcher::is_reg2reg_move(MachNode* m) {
 2580   ShouldNotReachHere();  // generic vector operands not supported
 2581   return false;
 2582 }
 2583 
 2584 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2585   ShouldNotReachHere();  // generic vector operands not supported
 2586   return false;
 2587 }
 2588 
 2589 // Return whether or not this register is ever used as an argument.
 2590 // This function is used on startup to build the trampoline stubs in
 2591 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2592 // call in the trampoline, and arguments in those registers not be
 2593 // available to the callee.
 2594 bool Matcher::can_be_java_arg(int reg)
 2595 {
 2596   return
 2597     reg ==  R0_num || reg == R0_H_num ||
 2598     reg ==  R1_num || reg == R1_H_num ||
 2599     reg ==  R2_num || reg == R2_H_num ||
 2600     reg ==  R3_num || reg == R3_H_num ||
 2601     reg ==  R4_num || reg == R4_H_num ||
 2602     reg ==  R5_num || reg == R5_H_num ||
 2603     reg ==  R6_num || reg == R6_H_num ||
 2604     reg ==  R7_num || reg == R7_H_num ||
 2605     reg ==  V0_num || reg == V0_H_num ||
 2606     reg ==  V1_num || reg == V1_H_num ||
 2607     reg ==  V2_num || reg == V2_H_num ||
 2608     reg ==  V3_num || reg == V3_H_num ||
 2609     reg ==  V4_num || reg == V4_H_num ||
 2610     reg ==  V5_num || reg == V5_H_num ||
 2611     reg ==  V6_num || reg == V6_H_num ||
 2612     reg ==  V7_num || reg == V7_H_num;
 2613 }
 2614 
 2615 bool Matcher::is_spillable_arg(int reg)
 2616 {
 2617   return can_be_java_arg(reg);
 2618 }
 2619 
 2620 uint Matcher::int_pressure_limit()
 2621 {
 2622   // JDK-8183543: When taking the number of available registers as int
 2623   // register pressure threshold, the jtreg test:
 2624   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2625   // failed due to C2 compilation failure with
 2626   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2627   //
 2628   // A derived pointer is live at CallNode and then is flagged by RA
 2629   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2630   // derived pointers and lastly fail to spill after reaching maximum
 2631   // number of iterations. Lowering the default pressure threshold to
 2632   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2633   // a high register pressure area of the code so that split_DEF can
 2634   // generate DefinitionSpillCopy for the derived pointer.
 2635   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2636   if (!PreserveFramePointer) {
 2637     // When PreserveFramePointer is off, frame pointer is allocatable,
 2638     // but different from other SOC registers, it is excluded from
 2639     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2640     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2641     // See check_pressure_at_fatproj().
 2642     default_int_pressure_threshold--;
 2643   }
 2644   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2645 }
 2646 
 2647 uint Matcher::float_pressure_limit()
 2648 {
 2649   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2650   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2651 }
 2652 
 2653 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2654   return false;
 2655 }
 2656 
 2657 RegMask Matcher::divI_proj_mask() {
 2658   ShouldNotReachHere();
 2659   return RegMask();
 2660 }
 2661 
 2662 // Register for MODI projection of divmodI.
 2663 RegMask Matcher::modI_proj_mask() {
 2664   ShouldNotReachHere();
 2665   return RegMask();
 2666 }
 2667 
 2668 // Register for DIVL projection of divmodL.
 2669 RegMask Matcher::divL_proj_mask() {
 2670   ShouldNotReachHere();
 2671   return RegMask();
 2672 }
 2673 
 2674 // Register for MODL projection of divmodL.
 2675 RegMask Matcher::modL_proj_mask() {
 2676   ShouldNotReachHere();
 2677   return RegMask();
 2678 }
 2679 
 2680 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2681   return FP_REG_mask();
 2682 }
 2683 
 2684 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2685   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2686     Node* u = addp->fast_out(i);
 2687     if (u->is_LoadStore()) {
 2688       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2689       // instructions) only take register indirect as an operand, so
 2690       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2691       // must fail.
 2692       return false;
 2693     }
 2694     if (u->is_Mem()) {
 2695       int opsize = u->as_Mem()->memory_size();
 2696       assert(opsize > 0, "unexpected memory operand size");
 2697       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2698         return false;
 2699       }
 2700     }
 2701   }
 2702   return true;
 2703 }
 2704 
 2705 bool can_combine_with_imm(Node* binary_node, Node* replicate_node) {
 2706   if (UseSVE == 0 || !VectorNode::is_invariant_vector(replicate_node)){
 2707     return false;
 2708   }
 2709   Node* imm_node = replicate_node->in(1);
 2710   if (!imm_node->is_Con()) {
 2711     return false;
 2712   }
 2713 
 2714   const Type* t = imm_node->bottom_type();
 2715   if (!(t->isa_int() || t->isa_long())) {
 2716     return false;
 2717   }
 2718 
 2719   switch (binary_node->Opcode()) {
 2720   case Op_AndV:
 2721   case Op_OrV:
 2722   case Op_XorV: {
 2723     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(binary_node));
 2724     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2725     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2726   }
 2727   case Op_AddVB:
 2728     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2729   case Op_AddVS:
 2730   case Op_AddVI:
 2731     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2732   case Op_AddVL:
 2733     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2734   default:
 2735     return false;
 2736   }
 2737 }
 2738 
 2739 bool is_vector_arith_imm_pattern(Node* n, Node* m) {
 2740   if (n != NULL && m != NULL) {
 2741     return can_combine_with_imm(n, m);
 2742   }
 2743   return false;
 2744 }
 2745 
 2746 // Should the matcher clone input 'm' of node 'n'?
 2747 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2748   // ShiftV src (ShiftCntV con)
 2749   // Binary src (Replicate con)
 2750   if (is_vshift_con_pattern(n, m) || is_vector_arith_imm_pattern(n, m)) {
 2751     mstack.push(m, Visit);
 2752     return true;
 2753   }
 2754 
 2755   return false;
 2756 }
 2757 
 2758 // Should the Matcher clone shifts on addressing modes, expecting them
 2759 // to be subsumed into complex addressing expressions or compute them
 2760 // into registers?
 2761 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2762   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2763     return true;
 2764   }
 2765 
 2766   Node *off = m->in(AddPNode::Offset);
 2767   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2768       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2769       // Are there other uses besides address expressions?
 2770       !is_visited(off)) {
 2771     address_visited.set(off->_idx); // Flag as address_visited
 2772     mstack.push(off->in(2), Visit);
 2773     Node *conv = off->in(1);
 2774     if (conv->Opcode() == Op_ConvI2L &&
 2775         // Are there other uses besides address expressions?
 2776         !is_visited(conv)) {
 2777       address_visited.set(conv->_idx); // Flag as address_visited
 2778       mstack.push(conv->in(1), Pre_Visit);
 2779     } else {
 2780       mstack.push(conv, Pre_Visit);
 2781     }
 2782     address_visited.test_set(m->_idx); // Flag as address_visited
 2783     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2784     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2785     return true;
 2786   } else if (off->Opcode() == Op_ConvI2L &&
 2787              // Are there other uses besides address expressions?
 2788              !is_visited(off)) {
 2789     address_visited.test_set(m->_idx); // Flag as address_visited
 2790     address_visited.set(off->_idx); // Flag as address_visited
 2791     mstack.push(off->in(1), Pre_Visit);
 2792     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2793     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2794     return true;
 2795   }
 2796   return false;
 2797 }
 2798 
 2799 bool Parse::do_one_bytecode_targeted() {
 2800   return false;
 2801 }
 2802 
 2803 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2804   C2_MacroAssembler _masm(&cbuf);                                       \
 2805   {                                                                     \
 2806     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2807     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2808     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2809     __ INSN(REG, as_Register(BASE));                                    \
 2810   }
 2811 
 2812 
 2813 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2814   {
 2815     Address::extend scale;
 2816 
 2817     // Hooboy, this is fugly.  We need a way to communicate to the
 2818     // encoder that the index needs to be sign extended, so we have to
 2819     // enumerate all the cases.
 2820     switch (opcode) {
 2821     case INDINDEXSCALEDI2L:
 2822     case INDINDEXSCALEDI2LN:
 2823     case INDINDEXI2L:
 2824     case INDINDEXI2LN:
 2825       scale = Address::sxtw(size);
 2826       break;
 2827     default:
 2828       scale = Address::lsl(size);
 2829     }
 2830 
 2831     if (index == -1) {
 2832       return Address(base, disp);
 2833     } else {
 2834       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2835       return Address(base, as_Register(index), scale);
 2836     }
 2837   }
 2838 
 2839 
 2840 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2841 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2842 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2843 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2844                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2845 
 2846   // Used for all non-volatile memory accesses.  The use of
 2847   // $mem->opcode() to discover whether this pattern uses sign-extended
 2848   // offsets is something of a kludge.
 2849   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2850                         Register reg, int opcode,
 2851                         Register base, int index, int scale, int disp,
 2852                         int size_in_memory)
 2853   {
 2854     Address addr = mem2address(opcode, base, index, scale, disp);
 2855     if (addr.getMode() == Address::base_plus_offset) {
 2856       /* If we get an out-of-range offset it is a bug in the compiler,
 2857          so we assert here. */
 2858       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2859              "c2 compiler bug");
 2860       /* Fix up any out-of-range offsets. */
 2861       assert_different_registers(rscratch1, base);
 2862       assert_different_registers(rscratch1, reg);
 2863       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2864     }
 2865     (masm.*insn)(reg, addr);
 2866   }
 2867 
 2868   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2869                         FloatRegister reg, int opcode,
 2870                         Register base, int index, int size, int disp,
 2871                         int size_in_memory)
 2872   {
 2873     Address::extend scale;
 2874 
 2875     switch (opcode) {
 2876     case INDINDEXSCALEDI2L:
 2877     case INDINDEXSCALEDI2LN:
 2878       scale = Address::sxtw(size);
 2879       break;
 2880     default:
 2881       scale = Address::lsl(size);
 2882     }
 2883 
 2884     if (index == -1) {
 2885       /* If we get an out-of-range offset it is a bug in the compiler,
 2886          so we assert here. */
 2887       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2888       /* Fix up any out-of-range offsets. */
 2889       assert_different_registers(rscratch1, base);
 2890       Address addr = Address(base, disp);
 2891       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2892       (masm.*insn)(reg, addr);
 2893     } else {
 2894       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2895       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2896     }
 2897   }
 2898 
 2899   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2900                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2901                         int opcode, Register base, int index, int size, int disp)
 2902   {
 2903     if (index == -1) {
 2904       (masm.*insn)(reg, T, Address(base, disp));
 2905     } else {
 2906       assert(disp == 0, "unsupported address mode");
 2907       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2908     }
 2909   }
 2910 
 2911 %}
 2912 
 2913 
 2914 
 2915 //----------ENCODING BLOCK-----------------------------------------------------
 2916 // This block specifies the encoding classes used by the compiler to
 2917 // output byte streams.  Encoding classes are parameterized macros
 2918 // used by Machine Instruction Nodes in order to generate the bit
 2919 // encoding of the instruction.  Operands specify their base encoding
 2920 // interface with the interface keyword.  There are currently
 2921 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2922 // COND_INTER.  REG_INTER causes an operand to generate a function
 2923 // which returns its register number when queried.  CONST_INTER causes
 2924 // an operand to generate a function which returns the value of the
 2925 // constant when queried.  MEMORY_INTER causes an operand to generate
 2926 // four functions which return the Base Register, the Index Register,
 2927 // the Scale Value, and the Offset Value of the operand when queried.
 2928 // COND_INTER causes an operand to generate six functions which return
 2929 // the encoding code (ie - encoding bits for the instruction)
 2930 // associated with each basic boolean condition for a conditional
 2931 // instruction.
 2932 //
 2933 // Instructions specify two basic values for encoding.  Again, a
 2934 // function is available to check if the constant displacement is an
 2935 // oop. They use the ins_encode keyword to specify their encoding
 2936 // classes (which must be a sequence of enc_class names, and their
 2937 // parameters, specified in the encoding block), and they use the
 2938 // opcode keyword to specify, in order, their primary, secondary, and
 2939 // tertiary opcode.  Only the opcode sections which a particular
 2940 // instruction needs for encoding need to be specified.
 2941 encode %{
 2942   // Build emit functions for each basic byte or larger field in the
 2943   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2944   // from C++ code in the enc_class source block.  Emit functions will
 2945   // live in the main source block for now.  In future, we can
 2946   // generalize this by adding a syntax that specifies the sizes of
 2947   // fields in an order, so that the adlc can build the emit functions
 2948   // automagically
 2949 
 2950   // catch all for unimplemented encodings
 2951   enc_class enc_unimplemented %{
 2952     C2_MacroAssembler _masm(&cbuf);
 2953     __ unimplemented("C2 catch all");
 2954   %}
 2955 
 2956   // BEGIN Non-volatile memory access
 2957 
 2958   // This encoding class is generated automatically from ad_encode.m4.
 2959   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2960   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2961     Register dst_reg = as_Register($dst$$reg);
 2962     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2963                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2964   %}
 2965 
 2966   // This encoding class is generated automatically from ad_encode.m4.
 2967   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2968   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2969     Register dst_reg = as_Register($dst$$reg);
 2970     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2971                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2972   %}
 2973 
 2974   // This encoding class is generated automatically from ad_encode.m4.
 2975   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2976   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2977     Register dst_reg = as_Register($dst$$reg);
 2978     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2979                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2980   %}
 2981 
 2982   // This encoding class is generated automatically from ad_encode.m4.
 2983   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2984   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2985     Register dst_reg = as_Register($dst$$reg);
 2986     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2987                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2988   %}
 2989 
 2990   // This encoding class is generated automatically from ad_encode.m4.
 2991   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2992   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2993     Register dst_reg = as_Register($dst$$reg);
 2994     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2995                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2996   %}
 2997 
 2998   // This encoding class is generated automatically from ad_encode.m4.
 2999   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3000   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 3001     Register dst_reg = as_Register($dst$$reg);
 3002     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 3003                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3004   %}
 3005 
 3006   // This encoding class is generated automatically from ad_encode.m4.
 3007   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3008   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 3009     Register dst_reg = as_Register($dst$$reg);
 3010     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3011                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3012   %}
 3013 
 3014   // This encoding class is generated automatically from ad_encode.m4.
 3015   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3016   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 3017     Register dst_reg = as_Register($dst$$reg);
 3018     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3019                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3020   %}
 3021 
 3022   // This encoding class is generated automatically from ad_encode.m4.
 3023   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3024   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 3025     Register dst_reg = as_Register($dst$$reg);
 3026     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3027                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3028   %}
 3029 
 3030   // This encoding class is generated automatically from ad_encode.m4.
 3031   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3032   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 3033     Register dst_reg = as_Register($dst$$reg);
 3034     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3035                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3036   %}
 3037 
 3038   // This encoding class is generated automatically from ad_encode.m4.
 3039   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3040   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 3041     Register dst_reg = as_Register($dst$$reg);
 3042     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 3043                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3044   %}
 3045 
 3046   // This encoding class is generated automatically from ad_encode.m4.
 3047   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3048   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 3049     Register dst_reg = as_Register($dst$$reg);
 3050     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 3051                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3052   %}
 3053 
 3054   // This encoding class is generated automatically from ad_encode.m4.
 3055   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3056   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3057     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3058     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3059                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3060   %}
 3061 
 3062   // This encoding class is generated automatically from ad_encode.m4.
 3063   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3064   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3065     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3066     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3067                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3068   %}
 3069 
 3070   // This encoding class is generated automatically from ad_encode.m4.
 3071   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3072   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3073     Register src_reg = as_Register($src$$reg);
 3074     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 3075                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3076   %}
 3077 
 3078   // This encoding class is generated automatically from ad_encode.m4.
 3079   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3080   enc_class aarch64_enc_strb0(memory1 mem) %{
 3081     C2_MacroAssembler _masm(&cbuf);
 3082     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3083                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3084   %}
 3085 
 3086   // This encoding class is generated automatically from ad_encode.m4.
 3087   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3088   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3089     Register src_reg = as_Register($src$$reg);
 3090     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3091                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3092   %}
 3093 
 3094   // This encoding class is generated automatically from ad_encode.m4.
 3095   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3096   enc_class aarch64_enc_strh0(memory2 mem) %{
 3097     C2_MacroAssembler _masm(&cbuf);
 3098     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3099                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3100   %}
 3101 
 3102   // This encoding class is generated automatically from ad_encode.m4.
 3103   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3104   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3105     Register src_reg = as_Register($src$$reg);
 3106     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3107                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3108   %}
 3109 
 3110   // This encoding class is generated automatically from ad_encode.m4.
 3111   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3112   enc_class aarch64_enc_strw0(memory4 mem) %{
 3113     C2_MacroAssembler _masm(&cbuf);
 3114     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3115                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3116   %}
 3117 
 3118   // This encoding class is generated automatically from ad_encode.m4.
 3119   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3120   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3121     Register src_reg = as_Register($src$$reg);
 3122     // we sometimes get asked to store the stack pointer into the
 3123     // current thread -- we cannot do that directly on AArch64
 3124     if (src_reg == r31_sp) {
 3125       C2_MacroAssembler _masm(&cbuf);
 3126       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3127       __ mov(rscratch2, sp);
 3128       src_reg = rscratch2;
 3129     }
 3130     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3131                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3132   %}
 3133 
 3134   // This encoding class is generated automatically from ad_encode.m4.
 3135   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3136   enc_class aarch64_enc_str0(memory8 mem) %{
 3137     C2_MacroAssembler _masm(&cbuf);
 3138     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3139                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3140   %}
 3141 
 3142   // This encoding class is generated automatically from ad_encode.m4.
 3143   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3144   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3145     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3146     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3147                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3148   %}
 3149 
 3150   // This encoding class is generated automatically from ad_encode.m4.
 3151   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3152   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3153     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3154     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3155                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3156   %}
 3157 
 3158   // This encoding class is generated automatically from ad_encode.m4.
 3159   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3160   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3161       C2_MacroAssembler _masm(&cbuf);
 3162       __ membar(Assembler::StoreStore);
 3163       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3164                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3165   %}
 3166 
 3167   // END Non-volatile memory access
 3168 
 3169   // Vector loads and stores
 3170   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3171     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3172     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3173        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3174   %}
 3175 
 3176   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3177     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3178     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3179        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3180   %}
 3181 
 3182   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3183     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3184     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3185        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3186   %}
 3187 
 3188   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3189     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3190     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3191        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3192   %}
 3193 
 3194   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3195     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3196     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3197        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3198   %}
 3199 
 3200   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3201     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3202     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3203        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3204   %}
 3205 
 3206   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3207     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3208     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3209        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3210   %}
 3211 
 3212   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3213     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3214     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3215        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3216   %}
 3217 
 3218   // volatile loads and stores
 3219 
 3220   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3221     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3222                  rscratch1, stlrb);
 3223   %}
 3224 
 3225   enc_class aarch64_enc_stlrb0(memory mem) %{
 3226     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3227                  rscratch1, stlrb);
 3228   %}
 3229 
 3230   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3231     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3232                  rscratch1, stlrh);
 3233   %}
 3234 
 3235   enc_class aarch64_enc_stlrh0(memory mem) %{
 3236     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3237                  rscratch1, stlrh);
 3238   %}
 3239 
 3240   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3241     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3242                  rscratch1, stlrw);
 3243   %}
 3244 
 3245   enc_class aarch64_enc_stlrw0(memory mem) %{
 3246     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3247                  rscratch1, stlrw);
 3248   %}
 3249 
 3250   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3251     Register dst_reg = as_Register($dst$$reg);
 3252     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3253              rscratch1, ldarb);
 3254     __ sxtbw(dst_reg, dst_reg);
 3255   %}
 3256 
 3257   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3258     Register dst_reg = as_Register($dst$$reg);
 3259     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3260              rscratch1, ldarb);
 3261     __ sxtb(dst_reg, dst_reg);
 3262   %}
 3263 
 3264   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3265     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3266              rscratch1, ldarb);
 3267   %}
 3268 
 3269   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3270     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3271              rscratch1, ldarb);
 3272   %}
 3273 
 3274   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3275     Register dst_reg = as_Register($dst$$reg);
 3276     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3277              rscratch1, ldarh);
 3278     __ sxthw(dst_reg, dst_reg);
 3279   %}
 3280 
 3281   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3282     Register dst_reg = as_Register($dst$$reg);
 3283     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3284              rscratch1, ldarh);
 3285     __ sxth(dst_reg, dst_reg);
 3286   %}
 3287 
 3288   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3289     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3290              rscratch1, ldarh);
 3291   %}
 3292 
 3293   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3294     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3295              rscratch1, ldarh);
 3296   %}
 3297 
 3298   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3299     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3300              rscratch1, ldarw);
 3301   %}
 3302 
 3303   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3304     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3305              rscratch1, ldarw);
 3306   %}
 3307 
 3308   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3309     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3310              rscratch1, ldar);
 3311   %}
 3312 
 3313   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3314     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3315              rscratch1, ldarw);
 3316     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3317   %}
 3318 
 3319   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3320     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3321              rscratch1, ldar);
 3322     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3323   %}
 3324 
 3325   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3326     Register src_reg = as_Register($src$$reg);
 3327     // we sometimes get asked to store the stack pointer into the
 3328     // current thread -- we cannot do that directly on AArch64
 3329     if (src_reg == r31_sp) {
 3330       C2_MacroAssembler _masm(&cbuf);
 3331       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3332       __ mov(rscratch2, sp);
 3333       src_reg = rscratch2;
 3334     }
 3335     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3336                  rscratch1, stlr);
 3337   %}
 3338 
 3339   enc_class aarch64_enc_stlr0(memory mem) %{
 3340     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3341                  rscratch1, stlr);
 3342   %}
 3343 
 3344   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3345     {
 3346       C2_MacroAssembler _masm(&cbuf);
 3347       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3348       __ fmovs(rscratch2, src_reg);
 3349     }
 3350     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3351                  rscratch1, stlrw);
 3352   %}
 3353 
 3354   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3355     {
 3356       C2_MacroAssembler _masm(&cbuf);
 3357       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3358       __ fmovd(rscratch2, src_reg);
 3359     }
 3360     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3361                  rscratch1, stlr);
 3362   %}
 3363 
 3364   // synchronized read/update encodings
 3365 
 3366   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3367     C2_MacroAssembler _masm(&cbuf);
 3368     Register dst_reg = as_Register($dst$$reg);
 3369     Register base = as_Register($mem$$base);
 3370     int index = $mem$$index;
 3371     int scale = $mem$$scale;
 3372     int disp = $mem$$disp;
 3373     if (index == -1) {
 3374        if (disp != 0) {
 3375         __ lea(rscratch1, Address(base, disp));
 3376         __ ldaxr(dst_reg, rscratch1);
 3377       } else {
 3378         // TODO
 3379         // should we ever get anything other than this case?
 3380         __ ldaxr(dst_reg, base);
 3381       }
 3382     } else {
 3383       Register index_reg = as_Register(index);
 3384       if (disp == 0) {
 3385         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3386         __ ldaxr(dst_reg, rscratch1);
 3387       } else {
 3388         __ lea(rscratch1, Address(base, disp));
 3389         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3390         __ ldaxr(dst_reg, rscratch1);
 3391       }
 3392     }
 3393   %}
 3394 
 3395   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3396     C2_MacroAssembler _masm(&cbuf);
 3397     Register src_reg = as_Register($src$$reg);
 3398     Register base = as_Register($mem$$base);
 3399     int index = $mem$$index;
 3400     int scale = $mem$$scale;
 3401     int disp = $mem$$disp;
 3402     if (index == -1) {
 3403        if (disp != 0) {
 3404         __ lea(rscratch2, Address(base, disp));
 3405         __ stlxr(rscratch1, src_reg, rscratch2);
 3406       } else {
 3407         // TODO
 3408         // should we ever get anything other than this case?
 3409         __ stlxr(rscratch1, src_reg, base);
 3410       }
 3411     } else {
 3412       Register index_reg = as_Register(index);
 3413       if (disp == 0) {
 3414         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3415         __ stlxr(rscratch1, src_reg, rscratch2);
 3416       } else {
 3417         __ lea(rscratch2, Address(base, disp));
 3418         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3419         __ stlxr(rscratch1, src_reg, rscratch2);
 3420       }
 3421     }
 3422     __ cmpw(rscratch1, zr);
 3423   %}
 3424 
 3425   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3426     C2_MacroAssembler _masm(&cbuf);
 3427     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3428     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3429                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3430                /*weak*/ false, noreg);
 3431   %}
 3432 
 3433   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3434     C2_MacroAssembler _masm(&cbuf);
 3435     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3436     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3437                Assembler::word, /*acquire*/ false, /*release*/ true,
 3438                /*weak*/ false, noreg);
 3439   %}
 3440 
 3441   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3442     C2_MacroAssembler _masm(&cbuf);
 3443     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3444     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3445                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3446                /*weak*/ false, noreg);
 3447   %}
 3448 
 3449   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3450     C2_MacroAssembler _masm(&cbuf);
 3451     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3452     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3453                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3454                /*weak*/ false, noreg);
 3455   %}
 3456 
 3457 
 3458   // The only difference between aarch64_enc_cmpxchg and
 3459   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3460   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3461   // lock.
 3462   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3463     C2_MacroAssembler _masm(&cbuf);
 3464     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3465     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3466                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3467                /*weak*/ false, noreg);
 3468   %}
 3469 
 3470   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3471     C2_MacroAssembler _masm(&cbuf);
 3472     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3473     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3474                Assembler::word, /*acquire*/ true, /*release*/ true,
 3475                /*weak*/ false, noreg);
 3476   %}
 3477 
 3478   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3481     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3482                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3483                /*weak*/ false, noreg);
 3484   %}
 3485 
 3486   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3487     C2_MacroAssembler _masm(&cbuf);
 3488     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3489     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3490                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3491                /*weak*/ false, noreg);
 3492   %}
 3493 
 3494   // auxiliary used for CompareAndSwapX to set result register
 3495   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3496     C2_MacroAssembler _masm(&cbuf);
 3497     Register res_reg = as_Register($res$$reg);
 3498     __ cset(res_reg, Assembler::EQ);
 3499   %}
 3500 
 3501   // prefetch encodings
 3502 
 3503   enc_class aarch64_enc_prefetchw(memory mem) %{
 3504     C2_MacroAssembler _masm(&cbuf);
 3505     Register base = as_Register($mem$$base);
 3506     int index = $mem$$index;
 3507     int scale = $mem$$scale;
 3508     int disp = $mem$$disp;
 3509     if (index == -1) {
 3510       __ prfm(Address(base, disp), PSTL1KEEP);
 3511     } else {
 3512       Register index_reg = as_Register(index);
 3513       if (disp == 0) {
 3514         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3515       } else {
 3516         __ lea(rscratch1, Address(base, disp));
 3517 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3518       }
 3519     }
 3520   %}
 3521 
 3522   /// mov envcodings
 3523 
 3524   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3525     C2_MacroAssembler _masm(&cbuf);
 3526     uint32_t con = (uint32_t)$src$$constant;
 3527     Register dst_reg = as_Register($dst$$reg);
 3528     if (con == 0) {
 3529       __ movw(dst_reg, zr);
 3530     } else {
 3531       __ movw(dst_reg, con);
 3532     }
 3533   %}
 3534 
 3535   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3536     C2_MacroAssembler _masm(&cbuf);
 3537     Register dst_reg = as_Register($dst$$reg);
 3538     uint64_t con = (uint64_t)$src$$constant;
 3539     if (con == 0) {
 3540       __ mov(dst_reg, zr);
 3541     } else {
 3542       __ mov(dst_reg, con);
 3543     }
 3544   %}
 3545 
 3546   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3547     C2_MacroAssembler _masm(&cbuf);
 3548     Register dst_reg = as_Register($dst$$reg);
 3549     address con = (address)$src$$constant;
 3550     if (con == NULL || con == (address)1) {
 3551       ShouldNotReachHere();
 3552     } else {
 3553       relocInfo::relocType rtype = $src->constant_reloc();
 3554       if (rtype == relocInfo::oop_type) {
 3555         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3556       } else if (rtype == relocInfo::metadata_type) {
 3557         __ mov_metadata(dst_reg, (Metadata*)con);
 3558       } else {
 3559         assert(rtype == relocInfo::none, "unexpected reloc type");
 3560         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3561           __ mov(dst_reg, con);
 3562         } else {
 3563           uint64_t offset;
 3564           __ adrp(dst_reg, con, offset);
 3565           __ add(dst_reg, dst_reg, offset);
 3566         }
 3567       }
 3568     }
 3569   %}
 3570 
 3571   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3572     C2_MacroAssembler _masm(&cbuf);
 3573     Register dst_reg = as_Register($dst$$reg);
 3574     __ mov(dst_reg, zr);
 3575   %}
 3576 
 3577   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3578     C2_MacroAssembler _masm(&cbuf);
 3579     Register dst_reg = as_Register($dst$$reg);
 3580     __ mov(dst_reg, (uint64_t)1);
 3581   %}
 3582 
 3583   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3584     C2_MacroAssembler _masm(&cbuf);
 3585     __ load_byte_map_base($dst$$Register);
 3586   %}
 3587 
 3588   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3589     C2_MacroAssembler _masm(&cbuf);
 3590     Register dst_reg = as_Register($dst$$reg);
 3591     address con = (address)$src$$constant;
 3592     if (con == NULL) {
 3593       ShouldNotReachHere();
 3594     } else {
 3595       relocInfo::relocType rtype = $src->constant_reloc();
 3596       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3597       __ set_narrow_oop(dst_reg, (jobject)con);
 3598     }
 3599   %}
 3600 
 3601   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3602     C2_MacroAssembler _masm(&cbuf);
 3603     Register dst_reg = as_Register($dst$$reg);
 3604     __ mov(dst_reg, zr);
 3605   %}
 3606 
 3607   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3608     C2_MacroAssembler _masm(&cbuf);
 3609     Register dst_reg = as_Register($dst$$reg);
 3610     address con = (address)$src$$constant;
 3611     if (con == NULL) {
 3612       ShouldNotReachHere();
 3613     } else {
 3614       relocInfo::relocType rtype = $src->constant_reloc();
 3615       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3616       __ set_narrow_klass(dst_reg, (Klass *)con);
 3617     }
 3618   %}
 3619 
 3620   // arithmetic encodings
 3621 
 3622   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3623     C2_MacroAssembler _masm(&cbuf);
 3624     Register dst_reg = as_Register($dst$$reg);
 3625     Register src_reg = as_Register($src1$$reg);
 3626     int32_t con = (int32_t)$src2$$constant;
 3627     // add has primary == 0, subtract has primary == 1
 3628     if ($primary) { con = -con; }
 3629     if (con < 0) {
 3630       __ subw(dst_reg, src_reg, -con);
 3631     } else {
 3632       __ addw(dst_reg, src_reg, con);
 3633     }
 3634   %}
 3635 
 3636   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3637     C2_MacroAssembler _masm(&cbuf);
 3638     Register dst_reg = as_Register($dst$$reg);
 3639     Register src_reg = as_Register($src1$$reg);
 3640     int32_t con = (int32_t)$src2$$constant;
 3641     // add has primary == 0, subtract has primary == 1
 3642     if ($primary) { con = -con; }
 3643     if (con < 0) {
 3644       __ sub(dst_reg, src_reg, -con);
 3645     } else {
 3646       __ add(dst_reg, src_reg, con);
 3647     }
 3648   %}
 3649 
 3650   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3651     C2_MacroAssembler _masm(&cbuf);
 3652    Register dst_reg = as_Register($dst$$reg);
 3653    Register src1_reg = as_Register($src1$$reg);
 3654    Register src2_reg = as_Register($src2$$reg);
 3655     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3656   %}
 3657 
 3658   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3659     C2_MacroAssembler _masm(&cbuf);
 3660    Register dst_reg = as_Register($dst$$reg);
 3661    Register src1_reg = as_Register($src1$$reg);
 3662    Register src2_reg = as_Register($src2$$reg);
 3663     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3664   %}
 3665 
 3666   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3667     C2_MacroAssembler _masm(&cbuf);
 3668    Register dst_reg = as_Register($dst$$reg);
 3669    Register src1_reg = as_Register($src1$$reg);
 3670    Register src2_reg = as_Register($src2$$reg);
 3671     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3672   %}
 3673 
 3674   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3675     C2_MacroAssembler _masm(&cbuf);
 3676    Register dst_reg = as_Register($dst$$reg);
 3677    Register src1_reg = as_Register($src1$$reg);
 3678    Register src2_reg = as_Register($src2$$reg);
 3679     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3680   %}
 3681 
 3682   // compare instruction encodings
 3683 
 3684   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3685     C2_MacroAssembler _masm(&cbuf);
 3686     Register reg1 = as_Register($src1$$reg);
 3687     Register reg2 = as_Register($src2$$reg);
 3688     __ cmpw(reg1, reg2);
 3689   %}
 3690 
 3691   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3692     C2_MacroAssembler _masm(&cbuf);
 3693     Register reg = as_Register($src1$$reg);
 3694     int32_t val = $src2$$constant;
 3695     if (val >= 0) {
 3696       __ subsw(zr, reg, val);
 3697     } else {
 3698       __ addsw(zr, reg, -val);
 3699     }
 3700   %}
 3701 
 3702   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3703     C2_MacroAssembler _masm(&cbuf);
 3704     Register reg1 = as_Register($src1$$reg);
 3705     uint32_t val = (uint32_t)$src2$$constant;
 3706     __ movw(rscratch1, val);
 3707     __ cmpw(reg1, rscratch1);
 3708   %}
 3709 
 3710   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3711     C2_MacroAssembler _masm(&cbuf);
 3712     Register reg1 = as_Register($src1$$reg);
 3713     Register reg2 = as_Register($src2$$reg);
 3714     __ cmp(reg1, reg2);
 3715   %}
 3716 
 3717   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3718     C2_MacroAssembler _masm(&cbuf);
 3719     Register reg = as_Register($src1$$reg);
 3720     int64_t val = $src2$$constant;
 3721     if (val >= 0) {
 3722       __ subs(zr, reg, val);
 3723     } else if (val != -val) {
 3724       __ adds(zr, reg, -val);
 3725     } else {
 3726     // aargh, Long.MIN_VALUE is a special case
 3727       __ orr(rscratch1, zr, (uint64_t)val);
 3728       __ subs(zr, reg, rscratch1);
 3729     }
 3730   %}
 3731 
 3732   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3733     C2_MacroAssembler _masm(&cbuf);
 3734     Register reg1 = as_Register($src1$$reg);
 3735     uint64_t val = (uint64_t)$src2$$constant;
 3736     __ mov(rscratch1, val);
 3737     __ cmp(reg1, rscratch1);
 3738   %}
 3739 
 3740   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3741     C2_MacroAssembler _masm(&cbuf);
 3742     Register reg1 = as_Register($src1$$reg);
 3743     Register reg2 = as_Register($src2$$reg);
 3744     __ cmp(reg1, reg2);
 3745   %}
 3746 
 3747   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3748     C2_MacroAssembler _masm(&cbuf);
 3749     Register reg1 = as_Register($src1$$reg);
 3750     Register reg2 = as_Register($src2$$reg);
 3751     __ cmpw(reg1, reg2);
 3752   %}
 3753 
 3754   enc_class aarch64_enc_testp(iRegP src) %{
 3755     C2_MacroAssembler _masm(&cbuf);
 3756     Register reg = as_Register($src$$reg);
 3757     __ cmp(reg, zr);
 3758   %}
 3759 
 3760   enc_class aarch64_enc_testn(iRegN src) %{
 3761     C2_MacroAssembler _masm(&cbuf);
 3762     Register reg = as_Register($src$$reg);
 3763     __ cmpw(reg, zr);
 3764   %}
 3765 
 3766   enc_class aarch64_enc_b(label lbl) %{
 3767     C2_MacroAssembler _masm(&cbuf);
 3768     Label *L = $lbl$$label;
 3769     __ b(*L);
 3770   %}
 3771 
 3772   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3773     C2_MacroAssembler _masm(&cbuf);
 3774     Label *L = $lbl$$label;
 3775     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3776   %}
 3777 
 3778   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3779     C2_MacroAssembler _masm(&cbuf);
 3780     Label *L = $lbl$$label;
 3781     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3782   %}
 3783 
 3784   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3785   %{
 3786      Register sub_reg = as_Register($sub$$reg);
 3787      Register super_reg = as_Register($super$$reg);
 3788      Register temp_reg = as_Register($temp$$reg);
 3789      Register result_reg = as_Register($result$$reg);
 3790 
 3791      Label miss;
 3792      C2_MacroAssembler _masm(&cbuf);
 3793      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3794                                      NULL, &miss,
 3795                                      /*set_cond_codes:*/ true);
 3796      if ($primary) {
 3797        __ mov(result_reg, zr);
 3798      }
 3799      __ bind(miss);
 3800   %}
 3801 
 3802   enc_class aarch64_enc_java_static_call(method meth) %{
 3803     C2_MacroAssembler _masm(&cbuf);
 3804 
 3805     address addr = (address)$meth$$method;
 3806     address call;
 3807     if (!_method) {
 3808       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3809       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3810       if (call == NULL) {
 3811         ciEnv::current()->record_failure("CodeCache is full");
 3812         return;
 3813       }
 3814     } else {
 3815       int method_index = resolved_method_index(cbuf);
 3816       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3817                                                   : static_call_Relocation::spec(method_index);
 3818       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3819       if (call == NULL) {
 3820         ciEnv::current()->record_failure("CodeCache is full");
 3821         return;
 3822       }
 3823       // Emit stub for static call
 3824       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3825       if (stub == NULL) {
 3826         ciEnv::current()->record_failure("CodeCache is full");
 3827         return;
 3828       }
 3829     }
 3830 
 3831     // Only non uncommon_trap calls need to reinitialize ptrue.
 3832     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3833       __ reinitialize_ptrue();
 3834     }
 3835   %}
 3836 
 3837   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3838     C2_MacroAssembler _masm(&cbuf);
 3839     int method_index = resolved_method_index(cbuf);
 3840     address call = __ ic_call((address)$meth$$method, method_index);
 3841     if (call == NULL) {
 3842       ciEnv::current()->record_failure("CodeCache is full");
 3843       return;
 3844     } else if (Compile::current()->max_vector_size() > 0) {
 3845       __ reinitialize_ptrue();
 3846     }
 3847   %}
 3848 
 3849   enc_class aarch64_enc_call_epilog() %{
 3850     C2_MacroAssembler _masm(&cbuf);
 3851     if (VerifyStackAtCalls) {
 3852       // Check that stack depth is unchanged: find majik cookie on stack
 3853       __ call_Unimplemented();
 3854     }
 3855   %}
 3856 
 3857   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3858     C2_MacroAssembler _masm(&cbuf);
 3859 
 3860     // some calls to generated routines (arraycopy code) are scheduled
 3861     // by C2 as runtime calls. if so we can call them using a br (they
 3862     // will be in a reachable segment) otherwise we have to use a blr
 3863     // which loads the absolute address into a register.
 3864     address entry = (address)$meth$$method;
 3865     CodeBlob *cb = CodeCache::find_blob(entry);
 3866     if (cb) {
 3867       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3868       if (call == NULL) {
 3869         ciEnv::current()->record_failure("CodeCache is full");
 3870         return;
 3871       }
 3872     } else {
 3873       Label retaddr;
 3874       __ adr(rscratch2, retaddr);
 3875       __ lea(rscratch1, RuntimeAddress(entry));
 3876       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3877       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3878       __ blr(rscratch1);
 3879       __ bind(retaddr);
 3880       __ add(sp, sp, 2 * wordSize);
 3881     }
 3882     if (Compile::current()->max_vector_size() > 0) {
 3883       __ reinitialize_ptrue();
 3884     }
 3885   %}
 3886 
 3887   enc_class aarch64_enc_rethrow() %{
 3888     C2_MacroAssembler _masm(&cbuf);
 3889     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3890   %}
 3891 
 3892   enc_class aarch64_enc_ret() %{
 3893     C2_MacroAssembler _masm(&cbuf);
 3894 #ifdef ASSERT
 3895     if (Compile::current()->max_vector_size() > 0) {
 3896       __ verify_ptrue();
 3897     }
 3898 #endif
 3899     __ ret(lr);
 3900   %}
 3901 
 3902   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3903     C2_MacroAssembler _masm(&cbuf);
 3904     Register target_reg = as_Register($jump_target$$reg);
 3905     __ br(target_reg);
 3906   %}
 3907 
 3908   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3909     C2_MacroAssembler _masm(&cbuf);
 3910     Register target_reg = as_Register($jump_target$$reg);
 3911     // exception oop should be in r0
 3912     // ret addr has been popped into lr
 3913     // callee expects it in r3
 3914     __ mov(r3, lr);
 3915     __ br(target_reg);
 3916   %}
 3917 
 3918   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3919     C2_MacroAssembler _masm(&cbuf);
 3920     Register oop = as_Register($object$$reg);
 3921     Register box = as_Register($box$$reg);
 3922     Register disp_hdr = as_Register($tmp$$reg);
 3923     Register tmp = as_Register($tmp2$$reg);
 3924     Label cont;
 3925     Label object_has_monitor;
 3926     Label cas_failed;
 3927 
 3928     assert_different_registers(oop, box, tmp, disp_hdr);
 3929 
 3930     // Load markWord from object into displaced_header.
 3931     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3932 
 3933     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3934       __ load_klass(tmp, oop);
 3935       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3936       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3937       __ br(Assembler::NE, cont);
 3938     }
 3939 
 3940     // Check for existing monitor
 3941     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3942 
 3943     if (!UseHeavyMonitors) {
 3944       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3945       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3946 
 3947       // Initialize the box. (Must happen before we update the object mark!)
 3948       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3949 
 3950       // Compare object markWord with an unlocked value (tmp) and if
 3951       // equal exchange the stack address of our box with object markWord.
 3952       // On failure disp_hdr contains the possibly locked markWord.
 3953       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3954                  /*release*/ true, /*weak*/ false, disp_hdr);
 3955       __ br(Assembler::EQ, cont);
 3956 
 3957       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3958 
 3959       // If the compare-and-exchange succeeded, then we found an unlocked
 3960       // object, will have now locked it will continue at label cont
 3961 
 3962       __ bind(cas_failed);
 3963       // We did not see an unlocked object so try the fast recursive case.
 3964 
 3965       // Check if the owner is self by comparing the value in the
 3966       // markWord of object (disp_hdr) with the stack pointer.
 3967       __ mov(rscratch1, sp);
 3968       __ sub(disp_hdr, disp_hdr, rscratch1);
 3969       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3970       // If condition is true we are cont and hence we can store 0 as the
 3971       // displaced header in the box, which indicates that it is a recursive lock.
 3972       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3973       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3974     } else {
 3975       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3976     }
 3977     __ b(cont);
 3978 
 3979     // Handle existing monitor.
 3980     __ bind(object_has_monitor);
 3981 
 3982     // The object's monitor m is unlocked iff m->owner == NULL,
 3983     // otherwise m->owner may contain a thread or a stack address.
 3984     //
 3985     // Try to CAS m->owner from NULL to current thread.
 3986     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3987     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3988                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3989 
 3990     // Store a non-null value into the box to avoid looking like a re-entrant
 3991     // lock. The fast-path monitor unlock code checks for
 3992     // markWord::monitor_value so use markWord::unused_mark which has the
 3993     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3994     __ mov(tmp, (address)markWord::unused_mark().value());
 3995     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3996 
 3997     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3998 
 3999     __ cmp(rscratch1, rthread);
 4000     __ br(Assembler::NE, cont); // Check for recursive locking
 4001 
 4002     // Recursive lock case
 4003     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 4004     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 4005 
 4006     __ bind(cont);
 4007     // flag == EQ indicates success
 4008     // flag == NE indicates failure
 4009   %}
 4010 
 4011   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 4012     C2_MacroAssembler _masm(&cbuf);
 4013     Register oop = as_Register($object$$reg);
 4014     Register box = as_Register($box$$reg);
 4015     Register disp_hdr = as_Register($tmp$$reg);
 4016     Register tmp = as_Register($tmp2$$reg);
 4017     Label cont;
 4018     Label object_has_monitor;
 4019 
 4020     assert_different_registers(oop, box, tmp, disp_hdr);
 4021 
 4022     if (!UseHeavyMonitors) {
 4023       // Find the lock address and load the displaced header from the stack.
 4024       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 4025 
 4026       // If the displaced header is 0, we have a recursive unlock.
 4027       __ cmp(disp_hdr, zr);
 4028       __ br(Assembler::EQ, cont);
 4029     }
 4030 
 4031     // Handle existing monitor.
 4032     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 4033     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 4034 
 4035     if (!UseHeavyMonitors) {
 4036       // Check if it is still a light weight lock, this is is true if we
 4037       // see the stack address of the basicLock in the markWord of the
 4038       // object.
 4039 
 4040       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 4041                  /*release*/ true, /*weak*/ false, tmp);
 4042     } else {
 4043       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 4044     }
 4045     __ b(cont);
 4046 
 4047     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 4048 
 4049     // Handle existing monitor.
 4050     __ bind(object_has_monitor);
 4051     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 4052     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 4053     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4054 
 4055     Label notRecursive;
 4056     __ cbz(disp_hdr, notRecursive);
 4057 
 4058     // Recursive lock
 4059     __ sub(disp_hdr, disp_hdr, 1u);
 4060     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4061     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 4062     __ b(cont);
 4063 
 4064     __ bind(notRecursive);
 4065     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 4066     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 4067     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 4068     __ cmp(rscratch1, zr); // Sets flags for result
 4069     __ cbnz(rscratch1, cont);
 4070     // need a release store here
 4071     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 4072     __ stlr(zr, tmp); // set unowned
 4073 
 4074     __ bind(cont);
 4075     // flag == EQ indicates success
 4076     // flag == NE indicates failure
 4077   %}
 4078 
 4079 %}
 4080 
 4081 //----------FRAME--------------------------------------------------------------
 4082 // Definition of frame structure and management information.
 4083 //
 4084 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4085 //                             |   (to get allocators register number
 4086 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4087 //  r   CALLER     |        |
 4088 //  o     |        +--------+      pad to even-align allocators stack-slot
 4089 //  w     V        |  pad0  |        numbers; owned by CALLER
 4090 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4091 //  h     ^        |   in   |  5
 4092 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4093 //  |     |        |        |  3
 4094 //  |     |        +--------+
 4095 //  V     |        | old out|      Empty on Intel, window on Sparc
 4096 //        |    old |preserve|      Must be even aligned.
 4097 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4098 //        |        |   in   |  3   area for Intel ret address
 4099 //     Owned by    |preserve|      Empty on Sparc.
 4100 //       SELF      +--------+
 4101 //        |        |  pad2  |  2   pad to align old SP
 4102 //        |        +--------+  1
 4103 //        |        | locks  |  0
 4104 //        |        +--------+----> OptoReg::stack0(), even aligned
 4105 //        |        |  pad1  | 11   pad to align new SP
 4106 //        |        +--------+
 4107 //        |        |        | 10
 4108 //        |        | spills |  9   spills
 4109 //        V        |        |  8   (pad0 slot for callee)
 4110 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4111 //        ^        |  out   |  7
 4112 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4113 //     Owned by    +--------+
 4114 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4115 //        |    new |preserve|      Must be even-aligned.
 4116 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4117 //        |        |        |
 4118 //
 4119 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4120 //         known from SELF's arguments and the Java calling convention.
 4121 //         Region 6-7 is determined per call site.
 4122 // Note 2: If the calling convention leaves holes in the incoming argument
 4123 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4124 //         are owned by the CALLEE.  Holes should not be necessary in the
 4125 //         incoming area, as the Java calling convention is completely under
 4126 //         the control of the AD file.  Doubles can be sorted and packed to
 4127 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 4128 //         varargs C calling conventions.
 4129 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4130 //         even aligned with pad0 as needed.
 4131 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4132 //           (the latter is true on Intel but is it false on AArch64?)
 4133 //         region 6-11 is even aligned; it may be padded out more so that
 4134 //         the region from SP to FP meets the minimum stack alignment.
 4135 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4136 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4137 //         SP meets the minimum alignment.
 4138 
 4139 frame %{
 4140   // These three registers define part of the calling convention
 4141   // between compiled code and the interpreter.
 4142 
 4143   // Inline Cache Register or Method for I2C.
 4144   inline_cache_reg(R12);
 4145 
 4146   // Number of stack slots consumed by locking an object
 4147   sync_stack_slots(2);
 4148 
 4149   // Compiled code's Frame Pointer
 4150   frame_pointer(R31);
 4151 
 4152   // Interpreter stores its frame pointer in a register which is
 4153   // stored to the stack by I2CAdaptors.
 4154   // I2CAdaptors convert from interpreted java to compiled java.
 4155   interpreter_frame_pointer(R29);
 4156 
 4157   // Stack alignment requirement
 4158   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4159 
 4160   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4161   // for calls to C.  Supports the var-args backing area for register parms.
 4162   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4163 
 4164   // The after-PROLOG location of the return address.  Location of
 4165   // return address specifies a type (REG or STACK) and a number
 4166   // representing the register number (i.e. - use a register name) or
 4167   // stack slot.
 4168   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4169   // Otherwise, it is above the locks and verification slot and alignment word
 4170   // TODO this may well be correct but need to check why that - 2 is there
 4171   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4172   // which folds in the space used for monitors
 4173   return_addr(STACK - 2 +
 4174               align_up((Compile::current()->in_preserve_stack_slots() +
 4175                         Compile::current()->fixed_slots()),
 4176                        stack_alignment_in_slots()));
 4177 
 4178   // Location of compiled Java return values.  Same as C for now.
 4179   return_value
 4180   %{
 4181     // TODO do we allow ideal_reg == Op_RegN???
 4182     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4183            "only return normal values");
 4184 
 4185     static const int lo[Op_RegL + 1] = { // enum name
 4186       0,                                 // Op_Node
 4187       0,                                 // Op_Set
 4188       R0_num,                            // Op_RegN
 4189       R0_num,                            // Op_RegI
 4190       R0_num,                            // Op_RegP
 4191       V0_num,                            // Op_RegF
 4192       V0_num,                            // Op_RegD
 4193       R0_num                             // Op_RegL
 4194     };
 4195 
 4196     static const int hi[Op_RegL + 1] = { // enum name
 4197       0,                                 // Op_Node
 4198       0,                                 // Op_Set
 4199       OptoReg::Bad,                      // Op_RegN
 4200       OptoReg::Bad,                      // Op_RegI
 4201       R0_H_num,                          // Op_RegP
 4202       OptoReg::Bad,                      // Op_RegF
 4203       V0_H_num,                          // Op_RegD
 4204       R0_H_num                           // Op_RegL
 4205     };
 4206 
 4207     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4208   %}
 4209 %}
 4210 
 4211 //----------ATTRIBUTES---------------------------------------------------------
 4212 //----------Operand Attributes-------------------------------------------------
 4213 op_attrib op_cost(1);        // Required cost attribute
 4214 
 4215 //----------Instruction Attributes---------------------------------------------
 4216 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4217 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4218 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4219                                 // a non-matching short branch variant
 4220                                 // of some long branch?
 4221 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4222                                 // be a power of 2) specifies the
 4223                                 // alignment that some part of the
 4224                                 // instruction (not necessarily the
 4225                                 // start) requires.  If > 1, a
 4226                                 // compute_padding() function must be
 4227                                 // provided for the instruction
 4228 
 4229 //----------OPERANDS-----------------------------------------------------------
 4230 // Operand definitions must precede instruction definitions for correct parsing
 4231 // in the ADLC because operands constitute user defined types which are used in
 4232 // instruction definitions.
 4233 
 4234 //----------Simple Operands----------------------------------------------------
 4235 
 4236 // Integer operands 32 bit
 4237 // 32 bit immediate
 4238 operand immI()
 4239 %{
 4240   match(ConI);
 4241 
 4242   op_cost(0);
 4243   format %{ %}
 4244   interface(CONST_INTER);
 4245 %}
 4246 
 4247 // 32 bit zero
 4248 operand immI0()
 4249 %{
 4250   predicate(n->get_int() == 0);
 4251   match(ConI);
 4252 
 4253   op_cost(0);
 4254   format %{ %}
 4255   interface(CONST_INTER);
 4256 %}
 4257 
 4258 // 32 bit unit increment
 4259 operand immI_1()
 4260 %{
 4261   predicate(n->get_int() == 1);
 4262   match(ConI);
 4263 
 4264   op_cost(0);
 4265   format %{ %}
 4266   interface(CONST_INTER);
 4267 %}
 4268 
 4269 // 32 bit unit decrement
 4270 operand immI_M1()
 4271 %{
 4272   predicate(n->get_int() == -1);
 4273   match(ConI);
 4274 
 4275   op_cost(0);
 4276   format %{ %}
 4277   interface(CONST_INTER);
 4278 %}
 4279 
 4280 // Shift values for add/sub extension shift
 4281 operand immIExt()
 4282 %{
 4283   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4284   match(ConI);
 4285 
 4286   op_cost(0);
 4287   format %{ %}
 4288   interface(CONST_INTER);
 4289 %}
 4290 
 4291 operand immI_gt_1()
 4292 %{
 4293   predicate(n->get_int() > 1);
 4294   match(ConI);
 4295 
 4296   op_cost(0);
 4297   format %{ %}
 4298   interface(CONST_INTER);
 4299 %}
 4300 
 4301 operand immI_le_4()
 4302 %{
 4303   predicate(n->get_int() <= 4);
 4304   match(ConI);
 4305 
 4306   op_cost(0);
 4307   format %{ %}
 4308   interface(CONST_INTER);
 4309 %}
 4310 
 4311 operand immI_31()
 4312 %{
 4313   predicate(n->get_int() == 31);
 4314   match(ConI);
 4315 
 4316   op_cost(0);
 4317   format %{ %}
 4318   interface(CONST_INTER);
 4319 %}
 4320 
 4321 operand immI_2()
 4322 %{
 4323   predicate(n->get_int() == 2);
 4324   match(ConI);
 4325 
 4326   op_cost(0);
 4327   format %{ %}
 4328   interface(CONST_INTER);
 4329 %}
 4330 
 4331 operand immI_4()
 4332 %{
 4333   predicate(n->get_int() == 4);
 4334   match(ConI);
 4335 
 4336   op_cost(0);
 4337   format %{ %}
 4338   interface(CONST_INTER);
 4339 %}
 4340 
 4341 operand immI_8()
 4342 %{
 4343   predicate(n->get_int() == 8);
 4344   match(ConI);
 4345 
 4346   op_cost(0);
 4347   format %{ %}
 4348   interface(CONST_INTER);
 4349 %}
 4350 
 4351 operand immI_16()
 4352 %{
 4353   predicate(n->get_int() == 16);
 4354   match(ConI);
 4355 
 4356   op_cost(0);
 4357   format %{ %}
 4358   interface(CONST_INTER);
 4359 %}
 4360 
 4361 operand immI_24()
 4362 %{
 4363   predicate(n->get_int() == 24);
 4364   match(ConI);
 4365 
 4366   op_cost(0);
 4367   format %{ %}
 4368   interface(CONST_INTER);
 4369 %}
 4370 
 4371 operand immI_32()
 4372 %{
 4373   predicate(n->get_int() == 32);
 4374   match(ConI);
 4375 
 4376   op_cost(0);
 4377   format %{ %}
 4378   interface(CONST_INTER);
 4379 %}
 4380 
 4381 operand immI_48()
 4382 %{
 4383   predicate(n->get_int() == 48);
 4384   match(ConI);
 4385 
 4386   op_cost(0);
 4387   format %{ %}
 4388   interface(CONST_INTER);
 4389 %}
 4390 
 4391 operand immI_56()
 4392 %{
 4393   predicate(n->get_int() == 56);
 4394   match(ConI);
 4395 
 4396   op_cost(0);
 4397   format %{ %}
 4398   interface(CONST_INTER);
 4399 %}
 4400 
 4401 operand immI_63()
 4402 %{
 4403   predicate(n->get_int() == 63);
 4404   match(ConI);
 4405 
 4406   op_cost(0);
 4407   format %{ %}
 4408   interface(CONST_INTER);
 4409 %}
 4410 
 4411 operand immI_64()
 4412 %{
 4413   predicate(n->get_int() == 64);
 4414   match(ConI);
 4415 
 4416   op_cost(0);
 4417   format %{ %}
 4418   interface(CONST_INTER);
 4419 %}
 4420 
 4421 operand immI_255()
 4422 %{
 4423   predicate(n->get_int() == 255);
 4424   match(ConI);
 4425 
 4426   op_cost(0);
 4427   format %{ %}
 4428   interface(CONST_INTER);
 4429 %}
 4430 
 4431 operand immI_65535()
 4432 %{
 4433   predicate(n->get_int() == 65535);
 4434   match(ConI);
 4435 
 4436   op_cost(0);
 4437   format %{ %}
 4438   interface(CONST_INTER);
 4439 %}
 4440 
 4441 operand immL_255()
 4442 %{
 4443   predicate(n->get_long() == 255L);
 4444   match(ConL);
 4445 
 4446   op_cost(0);
 4447   format %{ %}
 4448   interface(CONST_INTER);
 4449 %}
 4450 
 4451 operand immL_65535()
 4452 %{
 4453   predicate(n->get_long() == 65535L);
 4454   match(ConL);
 4455 
 4456   op_cost(0);
 4457   format %{ %}
 4458   interface(CONST_INTER);
 4459 %}
 4460 
 4461 operand immL_4294967295()
 4462 %{
 4463   predicate(n->get_long() == 4294967295L);
 4464   match(ConL);
 4465 
 4466   op_cost(0);
 4467   format %{ %}
 4468   interface(CONST_INTER);
 4469 %}
 4470 
 4471 operand immL_bitmask()
 4472 %{
 4473   predicate((n->get_long() != 0)
 4474             && ((n->get_long() & 0xc000000000000000l) == 0)
 4475             && is_power_of_2(n->get_long() + 1));
 4476   match(ConL);
 4477 
 4478   op_cost(0);
 4479   format %{ %}
 4480   interface(CONST_INTER);
 4481 %}
 4482 
 4483 operand immI_bitmask()
 4484 %{
 4485   predicate((n->get_int() != 0)
 4486             && ((n->get_int() & 0xc0000000) == 0)
 4487             && is_power_of_2(n->get_int() + 1));
 4488   match(ConI);
 4489 
 4490   op_cost(0);
 4491   format %{ %}
 4492   interface(CONST_INTER);
 4493 %}
 4494 
 4495 operand immL_positive_bitmaskI()
 4496 %{
 4497   predicate((n->get_long() != 0)
 4498             && ((julong)n->get_long() < 0x80000000ULL)
 4499             && is_power_of_2(n->get_long() + 1));
 4500   match(ConL);
 4501 
 4502   op_cost(0);
 4503   format %{ %}
 4504   interface(CONST_INTER);
 4505 %}
 4506 
 4507 // Scale values for scaled offset addressing modes (up to long but not quad)
 4508 operand immIScale()
 4509 %{
 4510   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4511   match(ConI);
 4512 
 4513   op_cost(0);
 4514   format %{ %}
 4515   interface(CONST_INTER);
 4516 %}
 4517 
 4518 // 26 bit signed offset -- for pc-relative branches
 4519 operand immI26()
 4520 %{
 4521   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4522   match(ConI);
 4523 
 4524   op_cost(0);
 4525   format %{ %}
 4526   interface(CONST_INTER);
 4527 %}
 4528 
 4529 // 19 bit signed offset -- for pc-relative loads
 4530 operand immI19()
 4531 %{
 4532   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4533   match(ConI);
 4534 
 4535   op_cost(0);
 4536   format %{ %}
 4537   interface(CONST_INTER);
 4538 %}
 4539 
 4540 // 12 bit unsigned offset -- for base plus immediate loads
 4541 operand immIU12()
 4542 %{
 4543   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4544   match(ConI);
 4545 
 4546   op_cost(0);
 4547   format %{ %}
 4548   interface(CONST_INTER);
 4549 %}
 4550 
 4551 operand immLU12()
 4552 %{
 4553   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4554   match(ConL);
 4555 
 4556   op_cost(0);
 4557   format %{ %}
 4558   interface(CONST_INTER);
 4559 %}
 4560 
 4561 // Offset for scaled or unscaled immediate loads and stores
 4562 operand immIOffset()
 4563 %{
 4564   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4565   match(ConI);
 4566 
 4567   op_cost(0);
 4568   format %{ %}
 4569   interface(CONST_INTER);
 4570 %}
 4571 
 4572 operand immIOffset1()
 4573 %{
 4574   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4575   match(ConI);
 4576 
 4577   op_cost(0);
 4578   format %{ %}
 4579   interface(CONST_INTER);
 4580 %}
 4581 
 4582 operand immIOffset2()
 4583 %{
 4584   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4585   match(ConI);
 4586 
 4587   op_cost(0);
 4588   format %{ %}
 4589   interface(CONST_INTER);
 4590 %}
 4591 
 4592 operand immIOffset4()
 4593 %{
 4594   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4595   match(ConI);
 4596 
 4597   op_cost(0);
 4598   format %{ %}
 4599   interface(CONST_INTER);
 4600 %}
 4601 
 4602 operand immIOffset8()
 4603 %{
 4604   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4605   match(ConI);
 4606 
 4607   op_cost(0);
 4608   format %{ %}
 4609   interface(CONST_INTER);
 4610 %}
 4611 
 4612 operand immIOffset16()
 4613 %{
 4614   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4615   match(ConI);
 4616 
 4617   op_cost(0);
 4618   format %{ %}
 4619   interface(CONST_INTER);
 4620 %}
 4621 
 4622 operand immLoffset()
 4623 %{
 4624   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4625   match(ConL);
 4626 
 4627   op_cost(0);
 4628   format %{ %}
 4629   interface(CONST_INTER);
 4630 %}
 4631 
 4632 operand immLoffset1()
 4633 %{
 4634   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4635   match(ConL);
 4636 
 4637   op_cost(0);
 4638   format %{ %}
 4639   interface(CONST_INTER);
 4640 %}
 4641 
 4642 operand immLoffset2()
 4643 %{
 4644   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4645   match(ConL);
 4646 
 4647   op_cost(0);
 4648   format %{ %}
 4649   interface(CONST_INTER);
 4650 %}
 4651 
 4652 operand immLoffset4()
 4653 %{
 4654   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4655   match(ConL);
 4656 
 4657   op_cost(0);
 4658   format %{ %}
 4659   interface(CONST_INTER);
 4660 %}
 4661 
 4662 operand immLoffset8()
 4663 %{
 4664   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4665   match(ConL);
 4666 
 4667   op_cost(0);
 4668   format %{ %}
 4669   interface(CONST_INTER);
 4670 %}
 4671 
 4672 operand immLoffset16()
 4673 %{
 4674   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4675   match(ConL);
 4676 
 4677   op_cost(0);
 4678   format %{ %}
 4679   interface(CONST_INTER);
 4680 %}
 4681 
 4682 // 8 bit signed value.
 4683 operand immI8()
 4684 %{
 4685   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4686   match(ConI);
 4687 
 4688   op_cost(0);
 4689   format %{ %}
 4690   interface(CONST_INTER);
 4691 %}
 4692 
 4693 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4694 operand immI8_shift8()
 4695 %{
 4696   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4697             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4698   match(ConI);
 4699 
 4700   op_cost(0);
 4701   format %{ %}
 4702   interface(CONST_INTER);
 4703 %}
 4704 
 4705 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4706 operand immL8_shift8()
 4707 %{
 4708   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4709             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4710   match(ConL);
 4711 
 4712   op_cost(0);
 4713   format %{ %}
 4714   interface(CONST_INTER);
 4715 %}
 4716 
 4717 // 8 bit integer valid for vector add sub immediate
 4718 operand immBAddSubV()
 4719 %{
 4720   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4721   match(ConI);
 4722 
 4723   op_cost(0);
 4724   format %{ %}
 4725   interface(CONST_INTER);
 4726 %}
 4727 
 4728 // 32 bit integer valid for add sub immediate
 4729 operand immIAddSub()
 4730 %{
 4731   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4732   match(ConI);
 4733   op_cost(0);
 4734   format %{ %}
 4735   interface(CONST_INTER);
 4736 %}
 4737 
 4738 // 32 bit integer valid for vector add sub immediate
 4739 operand immIAddSubV()
 4740 %{
 4741   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4742   match(ConI);
 4743 
 4744   op_cost(0);
 4745   format %{ %}
 4746   interface(CONST_INTER);
 4747 %}
 4748 
 4749 // 32 bit unsigned integer valid for logical immediate
 4750 
 4751 operand immBLog()
 4752 %{
 4753   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4754   match(ConI);
 4755 
 4756   op_cost(0);
 4757   format %{ %}
 4758   interface(CONST_INTER);
 4759 %}
 4760 
 4761 operand immSLog()
 4762 %{
 4763   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4764   match(ConI);
 4765 
 4766   op_cost(0);
 4767   format %{ %}
 4768   interface(CONST_INTER);
 4769 %}
 4770 
 4771 operand immILog()
 4772 %{
 4773   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4774   match(ConI);
 4775 
 4776   op_cost(0);
 4777   format %{ %}
 4778   interface(CONST_INTER);
 4779 %}
 4780 
 4781 // Integer operands 64 bit
 4782 // 64 bit immediate
 4783 operand immL()
 4784 %{
 4785   match(ConL);
 4786 
 4787   op_cost(0);
 4788   format %{ %}
 4789   interface(CONST_INTER);
 4790 %}
 4791 
 4792 // 64 bit zero
 4793 operand immL0()
 4794 %{
 4795   predicate(n->get_long() == 0);
 4796   match(ConL);
 4797 
 4798   op_cost(0);
 4799   format %{ %}
 4800   interface(CONST_INTER);
 4801 %}
 4802 
 4803 // 64 bit unit increment
 4804 operand immL_1()
 4805 %{
 4806   predicate(n->get_long() == 1);
 4807   match(ConL);
 4808 
 4809   op_cost(0);
 4810   format %{ %}
 4811   interface(CONST_INTER);
 4812 %}
 4813 
 4814 // 64 bit unit decrement
 4815 operand immL_M1()
 4816 %{
 4817   predicate(n->get_long() == -1);
 4818   match(ConL);
 4819 
 4820   op_cost(0);
 4821   format %{ %}
 4822   interface(CONST_INTER);
 4823 %}
 4824 
 4825 // 32 bit offset of pc in thread anchor
 4826 
 4827 operand immL_pc_off()
 4828 %{
 4829   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4830                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4831   match(ConL);
 4832 
 4833   op_cost(0);
 4834   format %{ %}
 4835   interface(CONST_INTER);
 4836 %}
 4837 
 4838 // 64 bit integer valid for add sub immediate
 4839 operand immLAddSub()
 4840 %{
 4841   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4842   match(ConL);
 4843   op_cost(0);
 4844   format %{ %}
 4845   interface(CONST_INTER);
 4846 %}
 4847 
 4848 // 64 bit integer valid for addv subv immediate
 4849 operand immLAddSubV()
 4850 %{
 4851   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4852   match(ConL);
 4853 
 4854   op_cost(0);
 4855   format %{ %}
 4856   interface(CONST_INTER);
 4857 %}
 4858 
 4859 // 64 bit integer valid for logical immediate
 4860 operand immLLog()
 4861 %{
 4862   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4863   match(ConL);
 4864   op_cost(0);
 4865   format %{ %}
 4866   interface(CONST_INTER);
 4867 %}
 4868 
 4869 // Long Immediate: low 32-bit mask
 4870 operand immL_32bits()
 4871 %{
 4872   predicate(n->get_long() == 0xFFFFFFFFL);
 4873   match(ConL);
 4874   op_cost(0);
 4875   format %{ %}
 4876   interface(CONST_INTER);
 4877 %}
 4878 
 4879 // Pointer operands
 4880 // Pointer Immediate
 4881 operand immP()
 4882 %{
 4883   match(ConP);
 4884 
 4885   op_cost(0);
 4886   format %{ %}
 4887   interface(CONST_INTER);
 4888 %}
 4889 
 4890 // NULL Pointer Immediate
 4891 operand immP0()
 4892 %{
 4893   predicate(n->get_ptr() == 0);
 4894   match(ConP);
 4895 
 4896   op_cost(0);
 4897   format %{ %}
 4898   interface(CONST_INTER);
 4899 %}
 4900 
 4901 // Pointer Immediate One
 4902 // this is used in object initialization (initial object header)
 4903 operand immP_1()
 4904 %{
 4905   predicate(n->get_ptr() == 1);
 4906   match(ConP);
 4907 
 4908   op_cost(0);
 4909   format %{ %}
 4910   interface(CONST_INTER);
 4911 %}
 4912 
 4913 // Card Table Byte Map Base
 4914 operand immByteMapBase()
 4915 %{
 4916   // Get base of card map
 4917   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4918             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4919   match(ConP);
 4920 
 4921   op_cost(0);
 4922   format %{ %}
 4923   interface(CONST_INTER);
 4924 %}
 4925 
 4926 // Pointer Immediate Minus One
 4927 // this is used when we want to write the current PC to the thread anchor
 4928 operand immP_M1()
 4929 %{
 4930   predicate(n->get_ptr() == -1);
 4931   match(ConP);
 4932 
 4933   op_cost(0);
 4934   format %{ %}
 4935   interface(CONST_INTER);
 4936 %}
 4937 
 4938 // Pointer Immediate Minus Two
 4939 // this is used when we want to write the current PC to the thread anchor
 4940 operand immP_M2()
 4941 %{
 4942   predicate(n->get_ptr() == -2);
 4943   match(ConP);
 4944 
 4945   op_cost(0);
 4946   format %{ %}
 4947   interface(CONST_INTER);
 4948 %}
 4949 
 4950 // Float and Double operands
 4951 // Double Immediate
 4952 operand immD()
 4953 %{
 4954   match(ConD);
 4955   op_cost(0);
 4956   format %{ %}
 4957   interface(CONST_INTER);
 4958 %}
 4959 
 4960 // Double Immediate: +0.0d
 4961 operand immD0()
 4962 %{
 4963   predicate(jlong_cast(n->getd()) == 0);
 4964   match(ConD);
 4965 
 4966   op_cost(0);
 4967   format %{ %}
 4968   interface(CONST_INTER);
 4969 %}
 4970 
 4971 // constant 'double +0.0'.
 4972 operand immDPacked()
 4973 %{
 4974   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4975   match(ConD);
 4976   op_cost(0);
 4977   format %{ %}
 4978   interface(CONST_INTER);
 4979 %}
 4980 
 4981 // Float Immediate
 4982 operand immF()
 4983 %{
 4984   match(ConF);
 4985   op_cost(0);
 4986   format %{ %}
 4987   interface(CONST_INTER);
 4988 %}
 4989 
 4990 // Float Immediate: +0.0f.
 4991 operand immF0()
 4992 %{
 4993   predicate(jint_cast(n->getf()) == 0);
 4994   match(ConF);
 4995 
 4996   op_cost(0);
 4997   format %{ %}
 4998   interface(CONST_INTER);
 4999 %}
 5000 
 5001 //
 5002 operand immFPacked()
 5003 %{
 5004   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 5005   match(ConF);
 5006   op_cost(0);
 5007   format %{ %}
 5008   interface(CONST_INTER);
 5009 %}
 5010 
 5011 // Narrow pointer operands
 5012 // Narrow Pointer Immediate
 5013 operand immN()
 5014 %{
 5015   match(ConN);
 5016 
 5017   op_cost(0);
 5018   format %{ %}
 5019   interface(CONST_INTER);
 5020 %}
 5021 
 5022 // Narrow NULL Pointer Immediate
 5023 operand immN0()
 5024 %{
 5025   predicate(n->get_narrowcon() == 0);
 5026   match(ConN);
 5027 
 5028   op_cost(0);
 5029   format %{ %}
 5030   interface(CONST_INTER);
 5031 %}
 5032 
 5033 operand immNKlass()
 5034 %{
 5035   match(ConNKlass);
 5036 
 5037   op_cost(0);
 5038   format %{ %}
 5039   interface(CONST_INTER);
 5040 %}
 5041 
 5042 // Integer 32 bit Register Operands
 5043 // Integer 32 bitRegister (excludes SP)
 5044 operand iRegI()
 5045 %{
 5046   constraint(ALLOC_IN_RC(any_reg32));
 5047   match(RegI);
 5048   match(iRegINoSp);
 5049   op_cost(0);
 5050   format %{ %}
 5051   interface(REG_INTER);
 5052 %}
 5053 
 5054 // Integer 32 bit Register not Special
 5055 operand iRegINoSp()
 5056 %{
 5057   constraint(ALLOC_IN_RC(no_special_reg32));
 5058   match(RegI);
 5059   op_cost(0);
 5060   format %{ %}
 5061   interface(REG_INTER);
 5062 %}
 5063 
 5064 // Integer 64 bit Register Operands
 5065 // Integer 64 bit Register (includes SP)
 5066 operand iRegL()
 5067 %{
 5068   constraint(ALLOC_IN_RC(any_reg));
 5069   match(RegL);
 5070   match(iRegLNoSp);
 5071   op_cost(0);
 5072   format %{ %}
 5073   interface(REG_INTER);
 5074 %}
 5075 
 5076 // Integer 64 bit Register not Special
 5077 operand iRegLNoSp()
 5078 %{
 5079   constraint(ALLOC_IN_RC(no_special_reg));
 5080   match(RegL);
 5081   match(iRegL_R0);
 5082   format %{ %}
 5083   interface(REG_INTER);
 5084 %}
 5085 
 5086 // Pointer Register Operands
 5087 // Pointer Register
 5088 operand iRegP()
 5089 %{
 5090   constraint(ALLOC_IN_RC(ptr_reg));
 5091   match(RegP);
 5092   match(iRegPNoSp);
 5093   match(iRegP_R0);
 5094   //match(iRegP_R2);
 5095   //match(iRegP_R4);
 5096   //match(iRegP_R5);
 5097   match(thread_RegP);
 5098   op_cost(0);
 5099   format %{ %}
 5100   interface(REG_INTER);
 5101 %}
 5102 
 5103 // Pointer 64 bit Register not Special
 5104 operand iRegPNoSp()
 5105 %{
 5106   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 5107   match(RegP);
 5108   // match(iRegP);
 5109   // match(iRegP_R0);
 5110   // match(iRegP_R2);
 5111   // match(iRegP_R4);
 5112   // match(iRegP_R5);
 5113   // match(thread_RegP);
 5114   op_cost(0);
 5115   format %{ %}
 5116   interface(REG_INTER);
 5117 %}
 5118 
 5119 // Pointer 64 bit Register R0 only
 5120 operand iRegP_R0()
 5121 %{
 5122   constraint(ALLOC_IN_RC(r0_reg));
 5123   match(RegP);
 5124   // match(iRegP);
 5125   match(iRegPNoSp);
 5126   op_cost(0);
 5127   format %{ %}
 5128   interface(REG_INTER);
 5129 %}
 5130 
 5131 // Pointer 64 bit Register R1 only
 5132 operand iRegP_R1()
 5133 %{
 5134   constraint(ALLOC_IN_RC(r1_reg));
 5135   match(RegP);
 5136   // match(iRegP);
 5137   match(iRegPNoSp);
 5138   op_cost(0);
 5139   format %{ %}
 5140   interface(REG_INTER);
 5141 %}
 5142 
 5143 // Pointer 64 bit Register R2 only
 5144 operand iRegP_R2()
 5145 %{
 5146   constraint(ALLOC_IN_RC(r2_reg));
 5147   match(RegP);
 5148   // match(iRegP);
 5149   match(iRegPNoSp);
 5150   op_cost(0);
 5151   format %{ %}
 5152   interface(REG_INTER);
 5153 %}
 5154 
 5155 // Pointer 64 bit Register R3 only
 5156 operand iRegP_R3()
 5157 %{
 5158   constraint(ALLOC_IN_RC(r3_reg));
 5159   match(RegP);
 5160   // match(iRegP);
 5161   match(iRegPNoSp);
 5162   op_cost(0);
 5163   format %{ %}
 5164   interface(REG_INTER);
 5165 %}
 5166 
 5167 // Pointer 64 bit Register R4 only
 5168 operand iRegP_R4()
 5169 %{
 5170   constraint(ALLOC_IN_RC(r4_reg));
 5171   match(RegP);
 5172   // match(iRegP);
 5173   match(iRegPNoSp);
 5174   op_cost(0);
 5175   format %{ %}
 5176   interface(REG_INTER);
 5177 %}
 5178 
 5179 // Pointer 64 bit Register R5 only
 5180 operand iRegP_R5()
 5181 %{
 5182   constraint(ALLOC_IN_RC(r5_reg));
 5183   match(RegP);
 5184   // match(iRegP);
 5185   match(iRegPNoSp);
 5186   op_cost(0);
 5187   format %{ %}
 5188   interface(REG_INTER);
 5189 %}
 5190 
 5191 // Pointer 64 bit Register R10 only
 5192 operand iRegP_R10()
 5193 %{
 5194   constraint(ALLOC_IN_RC(r10_reg));
 5195   match(RegP);
 5196   // match(iRegP);
 5197   match(iRegPNoSp);
 5198   op_cost(0);
 5199   format %{ %}
 5200   interface(REG_INTER);
 5201 %}
 5202 
 5203 // Long 64 bit Register R0 only
 5204 operand iRegL_R0()
 5205 %{
 5206   constraint(ALLOC_IN_RC(r0_reg));
 5207   match(RegL);
 5208   match(iRegLNoSp);
 5209   op_cost(0);
 5210   format %{ %}
 5211   interface(REG_INTER);
 5212 %}
 5213 
 5214 // Long 64 bit Register R2 only
 5215 operand iRegL_R2()
 5216 %{
 5217   constraint(ALLOC_IN_RC(r2_reg));
 5218   match(RegL);
 5219   match(iRegLNoSp);
 5220   op_cost(0);
 5221   format %{ %}
 5222   interface(REG_INTER);
 5223 %}
 5224 
 5225 // Long 64 bit Register R3 only
 5226 operand iRegL_R3()
 5227 %{
 5228   constraint(ALLOC_IN_RC(r3_reg));
 5229   match(RegL);
 5230   match(iRegLNoSp);
 5231   op_cost(0);
 5232   format %{ %}
 5233   interface(REG_INTER);
 5234 %}
 5235 
 5236 // Long 64 bit Register R11 only
 5237 operand iRegL_R11()
 5238 %{
 5239   constraint(ALLOC_IN_RC(r11_reg));
 5240   match(RegL);
 5241   match(iRegLNoSp);
 5242   op_cost(0);
 5243   format %{ %}
 5244   interface(REG_INTER);
 5245 %}
 5246 
 5247 // Pointer 64 bit Register FP only
 5248 operand iRegP_FP()
 5249 %{
 5250   constraint(ALLOC_IN_RC(fp_reg));
 5251   match(RegP);
 5252   // match(iRegP);
 5253   op_cost(0);
 5254   format %{ %}
 5255   interface(REG_INTER);
 5256 %}
 5257 
 5258 // Register R0 only
 5259 operand iRegI_R0()
 5260 %{
 5261   constraint(ALLOC_IN_RC(int_r0_reg));
 5262   match(RegI);
 5263   match(iRegINoSp);
 5264   op_cost(0);
 5265   format %{ %}
 5266   interface(REG_INTER);
 5267 %}
 5268 
 5269 // Register R2 only
 5270 operand iRegI_R2()
 5271 %{
 5272   constraint(ALLOC_IN_RC(int_r2_reg));
 5273   match(RegI);
 5274   match(iRegINoSp);
 5275   op_cost(0);
 5276   format %{ %}
 5277   interface(REG_INTER);
 5278 %}
 5279 
 5280 // Register R3 only
 5281 operand iRegI_R3()
 5282 %{
 5283   constraint(ALLOC_IN_RC(int_r3_reg));
 5284   match(RegI);
 5285   match(iRegINoSp);
 5286   op_cost(0);
 5287   format %{ %}
 5288   interface(REG_INTER);
 5289 %}
 5290 
 5291 
 5292 // Register R4 only
 5293 operand iRegI_R4()
 5294 %{
 5295   constraint(ALLOC_IN_RC(int_r4_reg));
 5296   match(RegI);
 5297   match(iRegINoSp);
 5298   op_cost(0);
 5299   format %{ %}
 5300   interface(REG_INTER);
 5301 %}
 5302 
 5303 
 5304 // Pointer Register Operands
 5305 // Narrow Pointer Register
 5306 operand iRegN()
 5307 %{
 5308   constraint(ALLOC_IN_RC(any_reg32));
 5309   match(RegN);
 5310   match(iRegNNoSp);
 5311   op_cost(0);
 5312   format %{ %}
 5313   interface(REG_INTER);
 5314 %}
 5315 
 5316 operand iRegN_R0()
 5317 %{
 5318   constraint(ALLOC_IN_RC(r0_reg));
 5319   match(iRegN);
 5320   op_cost(0);
 5321   format %{ %}
 5322   interface(REG_INTER);
 5323 %}
 5324 
 5325 operand iRegN_R2()
 5326 %{
 5327   constraint(ALLOC_IN_RC(r2_reg));
 5328   match(iRegN);
 5329   op_cost(0);
 5330   format %{ %}
 5331   interface(REG_INTER);
 5332 %}
 5333 
 5334 operand iRegN_R3()
 5335 %{
 5336   constraint(ALLOC_IN_RC(r3_reg));
 5337   match(iRegN);
 5338   op_cost(0);
 5339   format %{ %}
 5340   interface(REG_INTER);
 5341 %}
 5342 
 5343 // Integer 64 bit Register not Special
 5344 operand iRegNNoSp()
 5345 %{
 5346   constraint(ALLOC_IN_RC(no_special_reg32));
 5347   match(RegN);
 5348   op_cost(0);
 5349   format %{ %}
 5350   interface(REG_INTER);
 5351 %}
 5352 
 5353 // heap base register -- used for encoding immN0
 5354 
 5355 operand iRegIHeapbase()
 5356 %{
 5357   constraint(ALLOC_IN_RC(heapbase_reg));
 5358   match(RegI);
 5359   op_cost(0);
 5360   format %{ %}
 5361   interface(REG_INTER);
 5362 %}
 5363 
 5364 // Float Register
 5365 // Float register operands
 5366 operand vRegF()
 5367 %{
 5368   constraint(ALLOC_IN_RC(float_reg));
 5369   match(RegF);
 5370 
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 // Double Register
 5377 // Double register operands
 5378 operand vRegD()
 5379 %{
 5380   constraint(ALLOC_IN_RC(double_reg));
 5381   match(RegD);
 5382 
 5383   op_cost(0);
 5384   format %{ %}
 5385   interface(REG_INTER);
 5386 %}
 5387 
 5388 // Generic vector class. This will be used for
 5389 // all vector operands, including NEON and SVE,
 5390 // but currently only used for SVE VecA.
 5391 operand vReg()
 5392 %{
 5393   constraint(ALLOC_IN_RC(vectora_reg));
 5394   match(VecA);
 5395   op_cost(0);
 5396   format %{ %}
 5397   interface(REG_INTER);
 5398 %}
 5399 
 5400 operand vecD()
 5401 %{
 5402   constraint(ALLOC_IN_RC(vectord_reg));
 5403   match(VecD);
 5404 
 5405   op_cost(0);
 5406   format %{ %}
 5407   interface(REG_INTER);
 5408 %}
 5409 
 5410 operand vecX()
 5411 %{
 5412   constraint(ALLOC_IN_RC(vectorx_reg));
 5413   match(VecX);
 5414 
 5415   op_cost(0);
 5416   format %{ %}
 5417   interface(REG_INTER);
 5418 %}
 5419 
 5420 operand vRegD_V0()
 5421 %{
 5422   constraint(ALLOC_IN_RC(v0_reg));
 5423   match(RegD);
 5424   op_cost(0);
 5425   format %{ %}
 5426   interface(REG_INTER);
 5427 %}
 5428 
 5429 operand vRegD_V1()
 5430 %{
 5431   constraint(ALLOC_IN_RC(v1_reg));
 5432   match(RegD);
 5433   op_cost(0);
 5434   format %{ %}
 5435   interface(REG_INTER);
 5436 %}
 5437 
 5438 operand vRegD_V2()
 5439 %{
 5440   constraint(ALLOC_IN_RC(v2_reg));
 5441   match(RegD);
 5442   op_cost(0);
 5443   format %{ %}
 5444   interface(REG_INTER);
 5445 %}
 5446 
 5447 operand vRegD_V3()
 5448 %{
 5449   constraint(ALLOC_IN_RC(v3_reg));
 5450   match(RegD);
 5451   op_cost(0);
 5452   format %{ %}
 5453   interface(REG_INTER);
 5454 %}
 5455 
 5456 operand vRegD_V4()
 5457 %{
 5458   constraint(ALLOC_IN_RC(v4_reg));
 5459   match(RegD);
 5460   op_cost(0);
 5461   format %{ %}
 5462   interface(REG_INTER);
 5463 %}
 5464 
 5465 operand vRegD_V5()
 5466 %{
 5467   constraint(ALLOC_IN_RC(v5_reg));
 5468   match(RegD);
 5469   op_cost(0);
 5470   format %{ %}
 5471   interface(REG_INTER);
 5472 %}
 5473 
 5474 operand vRegD_V6()
 5475 %{
 5476   constraint(ALLOC_IN_RC(v6_reg));
 5477   match(RegD);
 5478   op_cost(0);
 5479   format %{ %}
 5480   interface(REG_INTER);
 5481 %}
 5482 
 5483 operand vRegD_V7()
 5484 %{
 5485   constraint(ALLOC_IN_RC(v7_reg));
 5486   match(RegD);
 5487   op_cost(0);
 5488   format %{ %}
 5489   interface(REG_INTER);
 5490 %}
 5491 
 5492 operand vRegD_V8()
 5493 %{
 5494   constraint(ALLOC_IN_RC(v8_reg));
 5495   match(RegD);
 5496   op_cost(0);
 5497   format %{ %}
 5498   interface(REG_INTER);
 5499 %}
 5500 
 5501 operand vRegD_V9()
 5502 %{
 5503   constraint(ALLOC_IN_RC(v9_reg));
 5504   match(RegD);
 5505   op_cost(0);
 5506   format %{ %}
 5507   interface(REG_INTER);
 5508 %}
 5509 
 5510 operand vRegD_V10()
 5511 %{
 5512   constraint(ALLOC_IN_RC(v10_reg));
 5513   match(RegD);
 5514   op_cost(0);
 5515   format %{ %}
 5516   interface(REG_INTER);
 5517 %}
 5518 
 5519 operand vRegD_V11()
 5520 %{
 5521   constraint(ALLOC_IN_RC(v11_reg));
 5522   match(RegD);
 5523   op_cost(0);
 5524   format %{ %}
 5525   interface(REG_INTER);
 5526 %}
 5527 
 5528 operand vRegD_V12()
 5529 %{
 5530   constraint(ALLOC_IN_RC(v12_reg));
 5531   match(RegD);
 5532   op_cost(0);
 5533   format %{ %}
 5534   interface(REG_INTER);
 5535 %}
 5536 
 5537 operand vRegD_V13()
 5538 %{
 5539   constraint(ALLOC_IN_RC(v13_reg));
 5540   match(RegD);
 5541   op_cost(0);
 5542   format %{ %}
 5543   interface(REG_INTER);
 5544 %}
 5545 
 5546 operand vRegD_V14()
 5547 %{
 5548   constraint(ALLOC_IN_RC(v14_reg));
 5549   match(RegD);
 5550   op_cost(0);
 5551   format %{ %}
 5552   interface(REG_INTER);
 5553 %}
 5554 
 5555 operand vRegD_V15()
 5556 %{
 5557   constraint(ALLOC_IN_RC(v15_reg));
 5558   match(RegD);
 5559   op_cost(0);
 5560   format %{ %}
 5561   interface(REG_INTER);
 5562 %}
 5563 
 5564 operand vRegD_V16()
 5565 %{
 5566   constraint(ALLOC_IN_RC(v16_reg));
 5567   match(RegD);
 5568   op_cost(0);
 5569   format %{ %}
 5570   interface(REG_INTER);
 5571 %}
 5572 
 5573 operand vRegD_V17()
 5574 %{
 5575   constraint(ALLOC_IN_RC(v17_reg));
 5576   match(RegD);
 5577   op_cost(0);
 5578   format %{ %}
 5579   interface(REG_INTER);
 5580 %}
 5581 
 5582 operand vRegD_V18()
 5583 %{
 5584   constraint(ALLOC_IN_RC(v18_reg));
 5585   match(RegD);
 5586   op_cost(0);
 5587   format %{ %}
 5588   interface(REG_INTER);
 5589 %}
 5590 
 5591 operand vRegD_V19()
 5592 %{
 5593   constraint(ALLOC_IN_RC(v19_reg));
 5594   match(RegD);
 5595   op_cost(0);
 5596   format %{ %}
 5597   interface(REG_INTER);
 5598 %}
 5599 
 5600 operand vRegD_V20()
 5601 %{
 5602   constraint(ALLOC_IN_RC(v20_reg));
 5603   match(RegD);
 5604   op_cost(0);
 5605   format %{ %}
 5606   interface(REG_INTER);
 5607 %}
 5608 
 5609 operand vRegD_V21()
 5610 %{
 5611   constraint(ALLOC_IN_RC(v21_reg));
 5612   match(RegD);
 5613   op_cost(0);
 5614   format %{ %}
 5615   interface(REG_INTER);
 5616 %}
 5617 
 5618 operand vRegD_V22()
 5619 %{
 5620   constraint(ALLOC_IN_RC(v22_reg));
 5621   match(RegD);
 5622   op_cost(0);
 5623   format %{ %}
 5624   interface(REG_INTER);
 5625 %}
 5626 
 5627 operand vRegD_V23()
 5628 %{
 5629   constraint(ALLOC_IN_RC(v23_reg));
 5630   match(RegD);
 5631   op_cost(0);
 5632   format %{ %}
 5633   interface(REG_INTER);
 5634 %}
 5635 
 5636 operand vRegD_V24()
 5637 %{
 5638   constraint(ALLOC_IN_RC(v24_reg));
 5639   match(RegD);
 5640   op_cost(0);
 5641   format %{ %}
 5642   interface(REG_INTER);
 5643 %}
 5644 
 5645 operand vRegD_V25()
 5646 %{
 5647   constraint(ALLOC_IN_RC(v25_reg));
 5648   match(RegD);
 5649   op_cost(0);
 5650   format %{ %}
 5651   interface(REG_INTER);
 5652 %}
 5653 
 5654 operand vRegD_V26()
 5655 %{
 5656   constraint(ALLOC_IN_RC(v26_reg));
 5657   match(RegD);
 5658   op_cost(0);
 5659   format %{ %}
 5660   interface(REG_INTER);
 5661 %}
 5662 
 5663 operand vRegD_V27()
 5664 %{
 5665   constraint(ALLOC_IN_RC(v27_reg));
 5666   match(RegD);
 5667   op_cost(0);
 5668   format %{ %}
 5669   interface(REG_INTER);
 5670 %}
 5671 
 5672 operand vRegD_V28()
 5673 %{
 5674   constraint(ALLOC_IN_RC(v28_reg));
 5675   match(RegD);
 5676   op_cost(0);
 5677   format %{ %}
 5678   interface(REG_INTER);
 5679 %}
 5680 
 5681 operand vRegD_V29()
 5682 %{
 5683   constraint(ALLOC_IN_RC(v29_reg));
 5684   match(RegD);
 5685   op_cost(0);
 5686   format %{ %}
 5687   interface(REG_INTER);
 5688 %}
 5689 
 5690 operand vRegD_V30()
 5691 %{
 5692   constraint(ALLOC_IN_RC(v30_reg));
 5693   match(RegD);
 5694   op_cost(0);
 5695   format %{ %}
 5696   interface(REG_INTER);
 5697 %}
 5698 
 5699 operand vRegD_V31()
 5700 %{
 5701   constraint(ALLOC_IN_RC(v31_reg));
 5702   match(RegD);
 5703   op_cost(0);
 5704   format %{ %}
 5705   interface(REG_INTER);
 5706 %}
 5707 
 5708 operand pReg()
 5709 %{
 5710   constraint(ALLOC_IN_RC(pr_reg));
 5711   match(RegVectMask);
 5712   match(pRegGov);
 5713   op_cost(0);
 5714   format %{ %}
 5715   interface(REG_INTER);
 5716 %}
 5717 
 5718 operand pRegGov()
 5719 %{
 5720   constraint(ALLOC_IN_RC(gov_pr));
 5721   match(RegVectMask);
 5722   op_cost(0);
 5723   format %{ %}
 5724   interface(REG_INTER);
 5725 %}
 5726 
 5727 // Flags register, used as output of signed compare instructions
 5728 
 5729 // note that on AArch64 we also use this register as the output for
 5730 // for floating point compare instructions (CmpF CmpD). this ensures
 5731 // that ordered inequality tests use GT, GE, LT or LE none of which
 5732 // pass through cases where the result is unordered i.e. one or both
 5733 // inputs to the compare is a NaN. this means that the ideal code can
 5734 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5735 // (where the comparison should always fail). EQ and NE tests are
 5736 // always generated in ideal code so that unordered folds into the NE
 5737 // case, matching the behaviour of AArch64 NE.
 5738 //
 5739 // This differs from x86 where the outputs of FP compares use a
 5740 // special FP flags registers and where compares based on this
 5741 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5742 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5743 // to explicitly handle the unordered case in branches. x86 also has
 5744 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5745 
 5746 operand rFlagsReg()
 5747 %{
 5748   constraint(ALLOC_IN_RC(int_flags));
 5749   match(RegFlags);
 5750 
 5751   op_cost(0);
 5752   format %{ "RFLAGS" %}
 5753   interface(REG_INTER);
 5754 %}
 5755 
 5756 // Flags register, used as output of unsigned compare instructions
 5757 operand rFlagsRegU()
 5758 %{
 5759   constraint(ALLOC_IN_RC(int_flags));
 5760   match(RegFlags);
 5761 
 5762   op_cost(0);
 5763   format %{ "RFLAGSU" %}
 5764   interface(REG_INTER);
 5765 %}
 5766 
 5767 // Special Registers
 5768 
 5769 // Method Register
 5770 operand inline_cache_RegP(iRegP reg)
 5771 %{
 5772   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5773   match(reg);
 5774   match(iRegPNoSp);
 5775   op_cost(0);
 5776   format %{ %}
 5777   interface(REG_INTER);
 5778 %}
 5779 
 5780 // Thread Register
 5781 operand thread_RegP(iRegP reg)
 5782 %{
 5783   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5784   match(reg);
 5785   op_cost(0);
 5786   format %{ %}
 5787   interface(REG_INTER);
 5788 %}
 5789 
 5790 operand lr_RegP(iRegP reg)
 5791 %{
 5792   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5793   match(reg);
 5794   op_cost(0);
 5795   format %{ %}
 5796   interface(REG_INTER);
 5797 %}
 5798 
 5799 //----------Memory Operands----------------------------------------------------
 5800 
 5801 operand indirect(iRegP reg)
 5802 %{
 5803   constraint(ALLOC_IN_RC(ptr_reg));
 5804   match(reg);
 5805   op_cost(0);
 5806   format %{ "[$reg]" %}
 5807   interface(MEMORY_INTER) %{
 5808     base($reg);
 5809     index(0xffffffff);
 5810     scale(0x0);
 5811     disp(0x0);
 5812   %}
 5813 %}
 5814 
 5815 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5816 %{
 5817   constraint(ALLOC_IN_RC(ptr_reg));
 5818   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5819   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5820   op_cost(0);
 5821   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5822   interface(MEMORY_INTER) %{
 5823     base($reg);
 5824     index($ireg);
 5825     scale($scale);
 5826     disp(0x0);
 5827   %}
 5828 %}
 5829 
 5830 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5831 %{
 5832   constraint(ALLOC_IN_RC(ptr_reg));
 5833   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5834   match(AddP reg (LShiftL lreg scale));
 5835   op_cost(0);
 5836   format %{ "$reg, $lreg lsl($scale)" %}
 5837   interface(MEMORY_INTER) %{
 5838     base($reg);
 5839     index($lreg);
 5840     scale($scale);
 5841     disp(0x0);
 5842   %}
 5843 %}
 5844 
 5845 operand indIndexI2L(iRegP reg, iRegI ireg)
 5846 %{
 5847   constraint(ALLOC_IN_RC(ptr_reg));
 5848   match(AddP reg (ConvI2L ireg));
 5849   op_cost(0);
 5850   format %{ "$reg, $ireg, 0, I2L" %}
 5851   interface(MEMORY_INTER) %{
 5852     base($reg);
 5853     index($ireg);
 5854     scale(0x0);
 5855     disp(0x0);
 5856   %}
 5857 %}
 5858 
 5859 operand indIndex(iRegP reg, iRegL lreg)
 5860 %{
 5861   constraint(ALLOC_IN_RC(ptr_reg));
 5862   match(AddP reg lreg);
 5863   op_cost(0);
 5864   format %{ "$reg, $lreg" %}
 5865   interface(MEMORY_INTER) %{
 5866     base($reg);
 5867     index($lreg);
 5868     scale(0x0);
 5869     disp(0x0);
 5870   %}
 5871 %}
 5872 
 5873 operand indOffI(iRegP reg, immIOffset off)
 5874 %{
 5875   constraint(ALLOC_IN_RC(ptr_reg));
 5876   match(AddP reg off);
 5877   op_cost(0);
 5878   format %{ "[$reg, $off]" %}
 5879   interface(MEMORY_INTER) %{
 5880     base($reg);
 5881     index(0xffffffff);
 5882     scale(0x0);
 5883     disp($off);
 5884   %}
 5885 %}
 5886 
 5887 operand indOffI1(iRegP reg, immIOffset1 off)
 5888 %{
 5889   constraint(ALLOC_IN_RC(ptr_reg));
 5890   match(AddP reg off);
 5891   op_cost(0);
 5892   format %{ "[$reg, $off]" %}
 5893   interface(MEMORY_INTER) %{
 5894     base($reg);
 5895     index(0xffffffff);
 5896     scale(0x0);
 5897     disp($off);
 5898   %}
 5899 %}
 5900 
 5901 operand indOffI2(iRegP reg, immIOffset2 off)
 5902 %{
 5903   constraint(ALLOC_IN_RC(ptr_reg));
 5904   match(AddP reg off);
 5905   op_cost(0);
 5906   format %{ "[$reg, $off]" %}
 5907   interface(MEMORY_INTER) %{
 5908     base($reg);
 5909     index(0xffffffff);
 5910     scale(0x0);
 5911     disp($off);
 5912   %}
 5913 %}
 5914 
 5915 operand indOffI4(iRegP reg, immIOffset4 off)
 5916 %{
 5917   constraint(ALLOC_IN_RC(ptr_reg));
 5918   match(AddP reg off);
 5919   op_cost(0);
 5920   format %{ "[$reg, $off]" %}
 5921   interface(MEMORY_INTER) %{
 5922     base($reg);
 5923     index(0xffffffff);
 5924     scale(0x0);
 5925     disp($off);
 5926   %}
 5927 %}
 5928 
 5929 operand indOffI8(iRegP reg, immIOffset8 off)
 5930 %{
 5931   constraint(ALLOC_IN_RC(ptr_reg));
 5932   match(AddP reg off);
 5933   op_cost(0);
 5934   format %{ "[$reg, $off]" %}
 5935   interface(MEMORY_INTER) %{
 5936     base($reg);
 5937     index(0xffffffff);
 5938     scale(0x0);
 5939     disp($off);
 5940   %}
 5941 %}
 5942 
 5943 operand indOffI16(iRegP reg, immIOffset16 off)
 5944 %{
 5945   constraint(ALLOC_IN_RC(ptr_reg));
 5946   match(AddP reg off);
 5947   op_cost(0);
 5948   format %{ "[$reg, $off]" %}
 5949   interface(MEMORY_INTER) %{
 5950     base($reg);
 5951     index(0xffffffff);
 5952     scale(0x0);
 5953     disp($off);
 5954   %}
 5955 %}
 5956 
 5957 operand indOffL(iRegP reg, immLoffset off)
 5958 %{
 5959   constraint(ALLOC_IN_RC(ptr_reg));
 5960   match(AddP reg off);
 5961   op_cost(0);
 5962   format %{ "[$reg, $off]" %}
 5963   interface(MEMORY_INTER) %{
 5964     base($reg);
 5965     index(0xffffffff);
 5966     scale(0x0);
 5967     disp($off);
 5968   %}
 5969 %}
 5970 
 5971 operand indOffL1(iRegP reg, immLoffset1 off)
 5972 %{
 5973   constraint(ALLOC_IN_RC(ptr_reg));
 5974   match(AddP reg off);
 5975   op_cost(0);
 5976   format %{ "[$reg, $off]" %}
 5977   interface(MEMORY_INTER) %{
 5978     base($reg);
 5979     index(0xffffffff);
 5980     scale(0x0);
 5981     disp($off);
 5982   %}
 5983 %}
 5984 
 5985 operand indOffL2(iRegP reg, immLoffset2 off)
 5986 %{
 5987   constraint(ALLOC_IN_RC(ptr_reg));
 5988   match(AddP reg off);
 5989   op_cost(0);
 5990   format %{ "[$reg, $off]" %}
 5991   interface(MEMORY_INTER) %{
 5992     base($reg);
 5993     index(0xffffffff);
 5994     scale(0x0);
 5995     disp($off);
 5996   %}
 5997 %}
 5998 
 5999 operand indOffL4(iRegP reg, immLoffset4 off)
 6000 %{
 6001   constraint(ALLOC_IN_RC(ptr_reg));
 6002   match(AddP reg off);
 6003   op_cost(0);
 6004   format %{ "[$reg, $off]" %}
 6005   interface(MEMORY_INTER) %{
 6006     base($reg);
 6007     index(0xffffffff);
 6008     scale(0x0);
 6009     disp($off);
 6010   %}
 6011 %}
 6012 
 6013 operand indOffL8(iRegP reg, immLoffset8 off)
 6014 %{
 6015   constraint(ALLOC_IN_RC(ptr_reg));
 6016   match(AddP reg off);
 6017   op_cost(0);
 6018   format %{ "[$reg, $off]" %}
 6019   interface(MEMORY_INTER) %{
 6020     base($reg);
 6021     index(0xffffffff);
 6022     scale(0x0);
 6023     disp($off);
 6024   %}
 6025 %}
 6026 
 6027 operand indOffL16(iRegP reg, immLoffset16 off)
 6028 %{
 6029   constraint(ALLOC_IN_RC(ptr_reg));
 6030   match(AddP reg off);
 6031   op_cost(0);
 6032   format %{ "[$reg, $off]" %}
 6033   interface(MEMORY_INTER) %{
 6034     base($reg);
 6035     index(0xffffffff);
 6036     scale(0x0);
 6037     disp($off);
 6038   %}
 6039 %}
 6040 
 6041 operand indirectN(iRegN reg)
 6042 %{
 6043   predicate(CompressedOops::shift() == 0);
 6044   constraint(ALLOC_IN_RC(ptr_reg));
 6045   match(DecodeN reg);
 6046   op_cost(0);
 6047   format %{ "[$reg]\t# narrow" %}
 6048   interface(MEMORY_INTER) %{
 6049     base($reg);
 6050     index(0xffffffff);
 6051     scale(0x0);
 6052     disp(0x0);
 6053   %}
 6054 %}
 6055 
 6056 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 6057 %{
 6058   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6059   constraint(ALLOC_IN_RC(ptr_reg));
 6060   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 6061   op_cost(0);
 6062   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 6063   interface(MEMORY_INTER) %{
 6064     base($reg);
 6065     index($ireg);
 6066     scale($scale);
 6067     disp(0x0);
 6068   %}
 6069 %}
 6070 
 6071 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 6072 %{
 6073   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6074   constraint(ALLOC_IN_RC(ptr_reg));
 6075   match(AddP (DecodeN reg) (LShiftL lreg scale));
 6076   op_cost(0);
 6077   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 6078   interface(MEMORY_INTER) %{
 6079     base($reg);
 6080     index($lreg);
 6081     scale($scale);
 6082     disp(0x0);
 6083   %}
 6084 %}
 6085 
 6086 operand indIndexI2LN(iRegN reg, iRegI ireg)
 6087 %{
 6088   predicate(CompressedOops::shift() == 0);
 6089   constraint(ALLOC_IN_RC(ptr_reg));
 6090   match(AddP (DecodeN reg) (ConvI2L ireg));
 6091   op_cost(0);
 6092   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 6093   interface(MEMORY_INTER) %{
 6094     base($reg);
 6095     index($ireg);
 6096     scale(0x0);
 6097     disp(0x0);
 6098   %}
 6099 %}
 6100 
 6101 operand indIndexN(iRegN reg, iRegL lreg)
 6102 %{
 6103   predicate(CompressedOops::shift() == 0);
 6104   constraint(ALLOC_IN_RC(ptr_reg));
 6105   match(AddP (DecodeN reg) lreg);
 6106   op_cost(0);
 6107   format %{ "$reg, $lreg\t# narrow" %}
 6108   interface(MEMORY_INTER) %{
 6109     base($reg);
 6110     index($lreg);
 6111     scale(0x0);
 6112     disp(0x0);
 6113   %}
 6114 %}
 6115 
 6116 operand indOffIN(iRegN reg, immIOffset off)
 6117 %{
 6118   predicate(CompressedOops::shift() == 0);
 6119   constraint(ALLOC_IN_RC(ptr_reg));
 6120   match(AddP (DecodeN reg) off);
 6121   op_cost(0);
 6122   format %{ "[$reg, $off]\t# narrow" %}
 6123   interface(MEMORY_INTER) %{
 6124     base($reg);
 6125     index(0xffffffff);
 6126     scale(0x0);
 6127     disp($off);
 6128   %}
 6129 %}
 6130 
 6131 operand indOffLN(iRegN reg, immLoffset off)
 6132 %{
 6133   predicate(CompressedOops::shift() == 0);
 6134   constraint(ALLOC_IN_RC(ptr_reg));
 6135   match(AddP (DecodeN reg) off);
 6136   op_cost(0);
 6137   format %{ "[$reg, $off]\t# narrow" %}
 6138   interface(MEMORY_INTER) %{
 6139     base($reg);
 6140     index(0xffffffff);
 6141     scale(0x0);
 6142     disp($off);
 6143   %}
 6144 %}
 6145 
 6146 
 6147 
 6148 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6149 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6150 %{
 6151   constraint(ALLOC_IN_RC(ptr_reg));
 6152   match(AddP reg off);
 6153   op_cost(0);
 6154   format %{ "[$reg, $off]" %}
 6155   interface(MEMORY_INTER) %{
 6156     base($reg);
 6157     index(0xffffffff);
 6158     scale(0x0);
 6159     disp($off);
 6160   %}
 6161 %}
 6162 
 6163 //----------Special Memory Operands--------------------------------------------
 6164 // Stack Slot Operand - This operand is used for loading and storing temporary
 6165 //                      values on the stack where a match requires a value to
 6166 //                      flow through memory.
 6167 operand stackSlotP(sRegP reg)
 6168 %{
 6169   constraint(ALLOC_IN_RC(stack_slots));
 6170   op_cost(100);
 6171   // No match rule because this operand is only generated in matching
 6172   // match(RegP);
 6173   format %{ "[$reg]" %}
 6174   interface(MEMORY_INTER) %{
 6175     base(0x1e);  // RSP
 6176     index(0x0);  // No Index
 6177     scale(0x0);  // No Scale
 6178     disp($reg);  // Stack Offset
 6179   %}
 6180 %}
 6181 
 6182 operand stackSlotI(sRegI reg)
 6183 %{
 6184   constraint(ALLOC_IN_RC(stack_slots));
 6185   // No match rule because this operand is only generated in matching
 6186   // match(RegI);
 6187   format %{ "[$reg]" %}
 6188   interface(MEMORY_INTER) %{
 6189     base(0x1e);  // RSP
 6190     index(0x0);  // No Index
 6191     scale(0x0);  // No Scale
 6192     disp($reg);  // Stack Offset
 6193   %}
 6194 %}
 6195 
 6196 operand stackSlotF(sRegF reg)
 6197 %{
 6198   constraint(ALLOC_IN_RC(stack_slots));
 6199   // No match rule because this operand is only generated in matching
 6200   // match(RegF);
 6201   format %{ "[$reg]" %}
 6202   interface(MEMORY_INTER) %{
 6203     base(0x1e);  // RSP
 6204     index(0x0);  // No Index
 6205     scale(0x0);  // No Scale
 6206     disp($reg);  // Stack Offset
 6207   %}
 6208 %}
 6209 
 6210 operand stackSlotD(sRegD reg)
 6211 %{
 6212   constraint(ALLOC_IN_RC(stack_slots));
 6213   // No match rule because this operand is only generated in matching
 6214   // match(RegD);
 6215   format %{ "[$reg]" %}
 6216   interface(MEMORY_INTER) %{
 6217     base(0x1e);  // RSP
 6218     index(0x0);  // No Index
 6219     scale(0x0);  // No Scale
 6220     disp($reg);  // Stack Offset
 6221   %}
 6222 %}
 6223 
 6224 operand stackSlotL(sRegL reg)
 6225 %{
 6226   constraint(ALLOC_IN_RC(stack_slots));
 6227   // No match rule because this operand is only generated in matching
 6228   // match(RegL);
 6229   format %{ "[$reg]" %}
 6230   interface(MEMORY_INTER) %{
 6231     base(0x1e);  // RSP
 6232     index(0x0);  // No Index
 6233     scale(0x0);  // No Scale
 6234     disp($reg);  // Stack Offset
 6235   %}
 6236 %}
 6237 
 6238 // Operands for expressing Control Flow
 6239 // NOTE: Label is a predefined operand which should not be redefined in
 6240 //       the AD file. It is generically handled within the ADLC.
 6241 
 6242 //----------Conditional Branch Operands----------------------------------------
 6243 // Comparison Op  - This is the operation of the comparison, and is limited to
 6244 //                  the following set of codes:
 6245 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6246 //
 6247 // Other attributes of the comparison, such as unsignedness, are specified
 6248 // by the comparison instruction that sets a condition code flags register.
 6249 // That result is represented by a flags operand whose subtype is appropriate
 6250 // to the unsignedness (etc.) of the comparison.
 6251 //
 6252 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6253 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6254 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6255 
 6256 // used for signed integral comparisons and fp comparisons
 6257 
 6258 operand cmpOp()
 6259 %{
 6260   match(Bool);
 6261 
 6262   format %{ "" %}
 6263   interface(COND_INTER) %{
 6264     equal(0x0, "eq");
 6265     not_equal(0x1, "ne");
 6266     less(0xb, "lt");
 6267     greater_equal(0xa, "ge");
 6268     less_equal(0xd, "le");
 6269     greater(0xc, "gt");
 6270     overflow(0x6, "vs");
 6271     no_overflow(0x7, "vc");
 6272   %}
 6273 %}
 6274 
 6275 // used for unsigned integral comparisons
 6276 
 6277 operand cmpOpU()
 6278 %{
 6279   match(Bool);
 6280 
 6281   format %{ "" %}
 6282   interface(COND_INTER) %{
 6283     equal(0x0, "eq");
 6284     not_equal(0x1, "ne");
 6285     less(0x3, "lo");
 6286     greater_equal(0x2, "hs");
 6287     less_equal(0x9, "ls");
 6288     greater(0x8, "hi");
 6289     overflow(0x6, "vs");
 6290     no_overflow(0x7, "vc");
 6291   %}
 6292 %}
 6293 
 6294 // used for certain integral comparisons which can be
 6295 // converted to cbxx or tbxx instructions
 6296 
 6297 operand cmpOpEqNe()
 6298 %{
 6299   match(Bool);
 6300   op_cost(0);
 6301   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6302             || n->as_Bool()->_test._test == BoolTest::eq);
 6303 
 6304   format %{ "" %}
 6305   interface(COND_INTER) %{
 6306     equal(0x0, "eq");
 6307     not_equal(0x1, "ne");
 6308     less(0xb, "lt");
 6309     greater_equal(0xa, "ge");
 6310     less_equal(0xd, "le");
 6311     greater(0xc, "gt");
 6312     overflow(0x6, "vs");
 6313     no_overflow(0x7, "vc");
 6314   %}
 6315 %}
 6316 
 6317 // used for certain integral comparisons which can be
 6318 // converted to cbxx or tbxx instructions
 6319 
 6320 operand cmpOpLtGe()
 6321 %{
 6322   match(Bool);
 6323   op_cost(0);
 6324 
 6325   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6326             || n->as_Bool()->_test._test == BoolTest::ge);
 6327 
 6328   format %{ "" %}
 6329   interface(COND_INTER) %{
 6330     equal(0x0, "eq");
 6331     not_equal(0x1, "ne");
 6332     less(0xb, "lt");
 6333     greater_equal(0xa, "ge");
 6334     less_equal(0xd, "le");
 6335     greater(0xc, "gt");
 6336     overflow(0x6, "vs");
 6337     no_overflow(0x7, "vc");
 6338   %}
 6339 %}
 6340 
 6341 // used for certain unsigned integral comparisons which can be
 6342 // converted to cbxx or tbxx instructions
 6343 
 6344 operand cmpOpUEqNeLtGe()
 6345 %{
 6346   match(Bool);
 6347   op_cost(0);
 6348 
 6349   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6350             || n->as_Bool()->_test._test == BoolTest::ne
 6351             || n->as_Bool()->_test._test == BoolTest::lt
 6352             || n->as_Bool()->_test._test == BoolTest::ge);
 6353 
 6354   format %{ "" %}
 6355   interface(COND_INTER) %{
 6356     equal(0x0, "eq");
 6357     not_equal(0x1, "ne");
 6358     less(0xb, "lt");
 6359     greater_equal(0xa, "ge");
 6360     less_equal(0xd, "le");
 6361     greater(0xc, "gt");
 6362     overflow(0x6, "vs");
 6363     no_overflow(0x7, "vc");
 6364   %}
 6365 %}
 6366 
 6367 // Special operand allowing long args to int ops to be truncated for free
 6368 
 6369 operand iRegL2I(iRegL reg) %{
 6370 
 6371   op_cost(0);
 6372 
 6373   match(ConvL2I reg);
 6374 
 6375   format %{ "l2i($reg)" %}
 6376 
 6377   interface(REG_INTER)
 6378 %}
 6379 
 6380 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6381 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6382 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6383 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6384 
 6385 //----------OPERAND CLASSES----------------------------------------------------
 6386 // Operand Classes are groups of operands that are used as to simplify
 6387 // instruction definitions by not requiring the AD writer to specify
 6388 // separate instructions for every form of operand when the
 6389 // instruction accepts multiple operand types with the same basic
 6390 // encoding and format. The classic case of this is memory operands.
 6391 
 6392 // memory is used to define read/write location for load/store
 6393 // instruction defs. we can turn a memory op into an Address
 6394 
 6395 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6396                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6397 
 6398 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6399                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6400 
 6401 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6402                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6403 
 6404 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6405                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6406 
 6407 // All of the memory operands. For the pipeline description.
 6408 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6409                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6410                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6411 
 6412 
 6413 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6414 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6415 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6416 // can be elided because the 32-bit instruction will just employ the
 6417 // lower 32 bits anyway.
 6418 //
 6419 // n.b. this does not elide all L2I conversions. if the truncated
 6420 // value is consumed by more than one operation then the ConvL2I
 6421 // cannot be bundled into the consuming nodes so an l2i gets planted
 6422 // (actually a movw $dst $src) and the downstream instructions consume
 6423 // the result of the l2i as an iRegI input. That's a shame since the
 6424 // movw is actually redundant but its not too costly.
 6425 
 6426 opclass iRegIorL2I(iRegI, iRegL2I);
 6427 
 6428 //----------PIPELINE-----------------------------------------------------------
 6429 // Rules which define the behavior of the target architectures pipeline.
 6430 
 6431 // For specific pipelines, eg A53, define the stages of that pipeline
 6432 //pipe_desc(ISS, EX1, EX2, WR);
 6433 #define ISS S0
 6434 #define EX1 S1
 6435 #define EX2 S2
 6436 #define WR  S3
 6437 
 6438 // Integer ALU reg operation
 6439 pipeline %{
 6440 
 6441 attributes %{
 6442   // ARM instructions are of fixed length
 6443   fixed_size_instructions;        // Fixed size instructions TODO does
 6444   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6445   // ARM instructions come in 32-bit word units
 6446   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6447   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6448   instruction_fetch_units = 1;       // of 64 bytes
 6449 
 6450   // List of nop instructions
 6451   nops( MachNop );
 6452 %}
 6453 
 6454 // We don't use an actual pipeline model so don't care about resources
 6455 // or description. we do use pipeline classes to introduce fixed
 6456 // latencies
 6457 
 6458 //----------RESOURCES----------------------------------------------------------
 6459 // Resources are the functional units available to the machine
 6460 
 6461 resources( INS0, INS1, INS01 = INS0 | INS1,
 6462            ALU0, ALU1, ALU = ALU0 | ALU1,
 6463            MAC,
 6464            DIV,
 6465            BRANCH,
 6466            LDST,
 6467            NEON_FP);
 6468 
 6469 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6470 // Pipeline Description specifies the stages in the machine's pipeline
 6471 
 6472 // Define the pipeline as a generic 6 stage pipeline
 6473 pipe_desc(S0, S1, S2, S3, S4, S5);
 6474 
 6475 //----------PIPELINE CLASSES---------------------------------------------------
 6476 // Pipeline Classes describe the stages in which input and output are
 6477 // referenced by the hardware pipeline.
 6478 
 6479 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6480 %{
 6481   single_instruction;
 6482   src1   : S1(read);
 6483   src2   : S2(read);
 6484   dst    : S5(write);
 6485   INS01  : ISS;
 6486   NEON_FP : S5;
 6487 %}
 6488 
 6489 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6490 %{
 6491   single_instruction;
 6492   src1   : S1(read);
 6493   src2   : S2(read);
 6494   dst    : S5(write);
 6495   INS01  : ISS;
 6496   NEON_FP : S5;
 6497 %}
 6498 
 6499 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6500 %{
 6501   single_instruction;
 6502   src    : S1(read);
 6503   dst    : S5(write);
 6504   INS01  : ISS;
 6505   NEON_FP : S5;
 6506 %}
 6507 
 6508 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6509 %{
 6510   single_instruction;
 6511   src    : S1(read);
 6512   dst    : S5(write);
 6513   INS01  : ISS;
 6514   NEON_FP : S5;
 6515 %}
 6516 
 6517 pipe_class fp_d2f(vRegF dst, vRegD src)
 6518 %{
 6519   single_instruction;
 6520   src    : S1(read);
 6521   dst    : S5(write);
 6522   INS01  : ISS;
 6523   NEON_FP : S5;
 6524 %}
 6525 
 6526 pipe_class fp_f2d(vRegD dst, vRegF src)
 6527 %{
 6528   single_instruction;
 6529   src    : S1(read);
 6530   dst    : S5(write);
 6531   INS01  : ISS;
 6532   NEON_FP : S5;
 6533 %}
 6534 
 6535 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6536 %{
 6537   single_instruction;
 6538   src    : S1(read);
 6539   dst    : S5(write);
 6540   INS01  : ISS;
 6541   NEON_FP : S5;
 6542 %}
 6543 
 6544 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6545 %{
 6546   single_instruction;
 6547   src    : S1(read);
 6548   dst    : S5(write);
 6549   INS01  : ISS;
 6550   NEON_FP : S5;
 6551 %}
 6552 
 6553 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6554 %{
 6555   single_instruction;
 6556   src    : S1(read);
 6557   dst    : S5(write);
 6558   INS01  : ISS;
 6559   NEON_FP : S5;
 6560 %}
 6561 
 6562 pipe_class fp_l2f(vRegF dst, iRegL src)
 6563 %{
 6564   single_instruction;
 6565   src    : S1(read);
 6566   dst    : S5(write);
 6567   INS01  : ISS;
 6568   NEON_FP : S5;
 6569 %}
 6570 
 6571 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6572 %{
 6573   single_instruction;
 6574   src    : S1(read);
 6575   dst    : S5(write);
 6576   INS01  : ISS;
 6577   NEON_FP : S5;
 6578 %}
 6579 
 6580 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6581 %{
 6582   single_instruction;
 6583   src    : S1(read);
 6584   dst    : S5(write);
 6585   INS01  : ISS;
 6586   NEON_FP : S5;
 6587 %}
 6588 
 6589 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6590 %{
 6591   single_instruction;
 6592   src    : S1(read);
 6593   dst    : S5(write);
 6594   INS01  : ISS;
 6595   NEON_FP : S5;
 6596 %}
 6597 
 6598 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6599 %{
 6600   single_instruction;
 6601   src    : S1(read);
 6602   dst    : S5(write);
 6603   INS01  : ISS;
 6604   NEON_FP : S5;
 6605 %}
 6606 
 6607 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6608 %{
 6609   single_instruction;
 6610   src1   : S1(read);
 6611   src2   : S2(read);
 6612   dst    : S5(write);
 6613   INS0   : ISS;
 6614   NEON_FP : S5;
 6615 %}
 6616 
 6617 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6618 %{
 6619   single_instruction;
 6620   src1   : S1(read);
 6621   src2   : S2(read);
 6622   dst    : S5(write);
 6623   INS0   : ISS;
 6624   NEON_FP : S5;
 6625 %}
 6626 
 6627 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6628 %{
 6629   single_instruction;
 6630   cr     : S1(read);
 6631   src1   : S1(read);
 6632   src2   : S1(read);
 6633   dst    : S3(write);
 6634   INS01  : ISS;
 6635   NEON_FP : S3;
 6636 %}
 6637 
 6638 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6639 %{
 6640   single_instruction;
 6641   cr     : S1(read);
 6642   src1   : S1(read);
 6643   src2   : S1(read);
 6644   dst    : S3(write);
 6645   INS01  : ISS;
 6646   NEON_FP : S3;
 6647 %}
 6648 
 6649 pipe_class fp_imm_s(vRegF dst)
 6650 %{
 6651   single_instruction;
 6652   dst    : S3(write);
 6653   INS01  : ISS;
 6654   NEON_FP : S3;
 6655 %}
 6656 
 6657 pipe_class fp_imm_d(vRegD dst)
 6658 %{
 6659   single_instruction;
 6660   dst    : S3(write);
 6661   INS01  : ISS;
 6662   NEON_FP : S3;
 6663 %}
 6664 
 6665 pipe_class fp_load_constant_s(vRegF dst)
 6666 %{
 6667   single_instruction;
 6668   dst    : S4(write);
 6669   INS01  : ISS;
 6670   NEON_FP : S4;
 6671 %}
 6672 
 6673 pipe_class fp_load_constant_d(vRegD dst)
 6674 %{
 6675   single_instruction;
 6676   dst    : S4(write);
 6677   INS01  : ISS;
 6678   NEON_FP : S4;
 6679 %}
 6680 
 6681 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6682 %{
 6683   single_instruction;
 6684   dst    : S5(write);
 6685   src1   : S1(read);
 6686   src2   : S1(read);
 6687   INS01  : ISS;
 6688   NEON_FP : S5;
 6689 %}
 6690 
 6691 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6692 %{
 6693   single_instruction;
 6694   dst    : S5(write);
 6695   src1   : S1(read);
 6696   src2   : S1(read);
 6697   INS0   : ISS;
 6698   NEON_FP : S5;
 6699 %}
 6700 
 6701 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6702 %{
 6703   single_instruction;
 6704   dst    : S5(write);
 6705   src1   : S1(read);
 6706   src2   : S1(read);
 6707   dst    : S1(read);
 6708   INS01  : ISS;
 6709   NEON_FP : S5;
 6710 %}
 6711 
 6712 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6713 %{
 6714   single_instruction;
 6715   dst    : S5(write);
 6716   src1   : S1(read);
 6717   src2   : S1(read);
 6718   dst    : S1(read);
 6719   INS0   : ISS;
 6720   NEON_FP : S5;
 6721 %}
 6722 
 6723 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6724 %{
 6725   single_instruction;
 6726   dst    : S4(write);
 6727   src1   : S2(read);
 6728   src2   : S2(read);
 6729   INS01  : ISS;
 6730   NEON_FP : S4;
 6731 %}
 6732 
 6733 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6734 %{
 6735   single_instruction;
 6736   dst    : S4(write);
 6737   src1   : S2(read);
 6738   src2   : S2(read);
 6739   INS0   : ISS;
 6740   NEON_FP : S4;
 6741 %}
 6742 
 6743 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6744 %{
 6745   single_instruction;
 6746   dst    : S3(write);
 6747   src1   : S2(read);
 6748   src2   : S2(read);
 6749   INS01  : ISS;
 6750   NEON_FP : S3;
 6751 %}
 6752 
 6753 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6754 %{
 6755   single_instruction;
 6756   dst    : S3(write);
 6757   src1   : S2(read);
 6758   src2   : S2(read);
 6759   INS0   : ISS;
 6760   NEON_FP : S3;
 6761 %}
 6762 
 6763 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6764 %{
 6765   single_instruction;
 6766   dst    : S3(write);
 6767   src    : S1(read);
 6768   shift  : S1(read);
 6769   INS01  : ISS;
 6770   NEON_FP : S3;
 6771 %}
 6772 
 6773 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6774 %{
 6775   single_instruction;
 6776   dst    : S3(write);
 6777   src    : S1(read);
 6778   shift  : S1(read);
 6779   INS0   : ISS;
 6780   NEON_FP : S3;
 6781 %}
 6782 
 6783 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6784 %{
 6785   single_instruction;
 6786   dst    : S3(write);
 6787   src    : S1(read);
 6788   INS01  : ISS;
 6789   NEON_FP : S3;
 6790 %}
 6791 
 6792 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6793 %{
 6794   single_instruction;
 6795   dst    : S3(write);
 6796   src    : S1(read);
 6797   INS0   : ISS;
 6798   NEON_FP : S3;
 6799 %}
 6800 
 6801 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6802 %{
 6803   single_instruction;
 6804   dst    : S5(write);
 6805   src1   : S1(read);
 6806   src2   : S1(read);
 6807   INS01  : ISS;
 6808   NEON_FP : S5;
 6809 %}
 6810 
 6811 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6812 %{
 6813   single_instruction;
 6814   dst    : S5(write);
 6815   src1   : S1(read);
 6816   src2   : S1(read);
 6817   INS0   : ISS;
 6818   NEON_FP : S5;
 6819 %}
 6820 
 6821 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6822 %{
 6823   single_instruction;
 6824   dst    : S5(write);
 6825   src1   : S1(read);
 6826   src2   : S1(read);
 6827   INS0   : ISS;
 6828   NEON_FP : S5;
 6829 %}
 6830 
 6831 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6832 %{
 6833   single_instruction;
 6834   dst    : S5(write);
 6835   src1   : S1(read);
 6836   src2   : S1(read);
 6837   INS0   : ISS;
 6838   NEON_FP : S5;
 6839 %}
 6840 
 6841 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6842 %{
 6843   single_instruction;
 6844   dst    : S5(write);
 6845   src    : S1(read);
 6846   INS0   : ISS;
 6847   NEON_FP : S5;
 6848 %}
 6849 
 6850 pipe_class vunop_fp64(vecD dst, vecD src)
 6851 %{
 6852   single_instruction;
 6853   dst    : S5(write);
 6854   src    : S1(read);
 6855   INS01  : ISS;
 6856   NEON_FP : S5;
 6857 %}
 6858 
 6859 pipe_class vunop_fp128(vecX dst, vecX src)
 6860 %{
 6861   single_instruction;
 6862   dst    : S5(write);
 6863   src    : S1(read);
 6864   INS0   : ISS;
 6865   NEON_FP : S5;
 6866 %}
 6867 
 6868 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6869 %{
 6870   single_instruction;
 6871   dst    : S3(write);
 6872   src    : S1(read);
 6873   INS01  : ISS;
 6874   NEON_FP : S3;
 6875 %}
 6876 
 6877 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6878 %{
 6879   single_instruction;
 6880   dst    : S3(write);
 6881   src    : S1(read);
 6882   INS01  : ISS;
 6883   NEON_FP : S3;
 6884 %}
 6885 
 6886 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6887 %{
 6888   single_instruction;
 6889   dst    : S3(write);
 6890   src    : S1(read);
 6891   INS01  : ISS;
 6892   NEON_FP : S3;
 6893 %}
 6894 
 6895 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6896 %{
 6897   single_instruction;
 6898   dst    : S3(write);
 6899   src    : S1(read);
 6900   INS01  : ISS;
 6901   NEON_FP : S3;
 6902 %}
 6903 
 6904 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6905 %{
 6906   single_instruction;
 6907   dst    : S3(write);
 6908   src    : S1(read);
 6909   INS01  : ISS;
 6910   NEON_FP : S3;
 6911 %}
 6912 
 6913 pipe_class vmovi_reg_imm64(vecD dst)
 6914 %{
 6915   single_instruction;
 6916   dst    : S3(write);
 6917   INS01  : ISS;
 6918   NEON_FP : S3;
 6919 %}
 6920 
 6921 pipe_class vmovi_reg_imm128(vecX dst)
 6922 %{
 6923   single_instruction;
 6924   dst    : S3(write);
 6925   INS0   : ISS;
 6926   NEON_FP : S3;
 6927 %}
 6928 
 6929 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6930 %{
 6931   single_instruction;
 6932   dst    : S5(write);
 6933   mem    : ISS(read);
 6934   INS01  : ISS;
 6935   NEON_FP : S3;
 6936 %}
 6937 
 6938 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6939 %{
 6940   single_instruction;
 6941   dst    : S5(write);
 6942   mem    : ISS(read);
 6943   INS01  : ISS;
 6944   NEON_FP : S3;
 6945 %}
 6946 
 6947 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6948 %{
 6949   single_instruction;
 6950   mem    : ISS(read);
 6951   src    : S2(read);
 6952   INS01  : ISS;
 6953   NEON_FP : S3;
 6954 %}
 6955 
 6956 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6957 %{
 6958   single_instruction;
 6959   mem    : ISS(read);
 6960   src    : S2(read);
 6961   INS01  : ISS;
 6962   NEON_FP : S3;
 6963 %}
 6964 
 6965 //------- Integer ALU operations --------------------------
 6966 
 6967 // Integer ALU reg-reg operation
 6968 // Operands needed in EX1, result generated in EX2
 6969 // Eg.  ADD     x0, x1, x2
 6970 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6971 %{
 6972   single_instruction;
 6973   dst    : EX2(write);
 6974   src1   : EX1(read);
 6975   src2   : EX1(read);
 6976   INS01  : ISS; // Dual issue as instruction 0 or 1
 6977   ALU    : EX2;
 6978 %}
 6979 
 6980 // Integer ALU reg-reg operation with constant shift
 6981 // Shifted register must be available in LATE_ISS instead of EX1
 6982 // Eg.  ADD     x0, x1, x2, LSL #2
 6983 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6984 %{
 6985   single_instruction;
 6986   dst    : EX2(write);
 6987   src1   : EX1(read);
 6988   src2   : ISS(read);
 6989   INS01  : ISS;
 6990   ALU    : EX2;
 6991 %}
 6992 
 6993 // Integer ALU reg operation with constant shift
 6994 // Eg.  LSL     x0, x1, #shift
 6995 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6996 %{
 6997   single_instruction;
 6998   dst    : EX2(write);
 6999   src1   : ISS(read);
 7000   INS01  : ISS;
 7001   ALU    : EX2;
 7002 %}
 7003 
 7004 // Integer ALU reg-reg operation with variable shift
 7005 // Both operands must be available in LATE_ISS instead of EX1
 7006 // Result is available in EX1 instead of EX2
 7007 // Eg.  LSLV    x0, x1, x2
 7008 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 7009 %{
 7010   single_instruction;
 7011   dst    : EX1(write);
 7012   src1   : ISS(read);
 7013   src2   : ISS(read);
 7014   INS01  : ISS;
 7015   ALU    : EX1;
 7016 %}
 7017 
 7018 // Integer ALU reg-reg operation with extract
 7019 // As for _vshift above, but result generated in EX2
 7020 // Eg.  EXTR    x0, x1, x2, #N
 7021 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 7022 %{
 7023   single_instruction;
 7024   dst    : EX2(write);
 7025   src1   : ISS(read);
 7026   src2   : ISS(read);
 7027   INS1   : ISS; // Can only dual issue as Instruction 1
 7028   ALU    : EX1;
 7029 %}
 7030 
 7031 // Integer ALU reg operation
 7032 // Eg.  NEG     x0, x1
 7033 pipe_class ialu_reg(iRegI dst, iRegI src)
 7034 %{
 7035   single_instruction;
 7036   dst    : EX2(write);
 7037   src    : EX1(read);
 7038   INS01  : ISS;
 7039   ALU    : EX2;
 7040 %}
 7041 
 7042 // Integer ALU reg mmediate operation
 7043 // Eg.  ADD     x0, x1, #N
 7044 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 7045 %{
 7046   single_instruction;
 7047   dst    : EX2(write);
 7048   src1   : EX1(read);
 7049   INS01  : ISS;
 7050   ALU    : EX2;
 7051 %}
 7052 
 7053 // Integer ALU immediate operation (no source operands)
 7054 // Eg.  MOV     x0, #N
 7055 pipe_class ialu_imm(iRegI dst)
 7056 %{
 7057   single_instruction;
 7058   dst    : EX1(write);
 7059   INS01  : ISS;
 7060   ALU    : EX1;
 7061 %}
 7062 
 7063 //------- Compare operation -------------------------------
 7064 
 7065 // Compare reg-reg
 7066 // Eg.  CMP     x0, x1
 7067 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 7068 %{
 7069   single_instruction;
 7070 //  fixed_latency(16);
 7071   cr     : EX2(write);
 7072   op1    : EX1(read);
 7073   op2    : EX1(read);
 7074   INS01  : ISS;
 7075   ALU    : EX2;
 7076 %}
 7077 
 7078 // Compare reg-reg
 7079 // Eg.  CMP     x0, #N
 7080 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 7081 %{
 7082   single_instruction;
 7083 //  fixed_latency(16);
 7084   cr     : EX2(write);
 7085   op1    : EX1(read);
 7086   INS01  : ISS;
 7087   ALU    : EX2;
 7088 %}
 7089 
 7090 //------- Conditional instructions ------------------------
 7091 
 7092 // Conditional no operands
 7093 // Eg.  CSINC   x0, zr, zr, <cond>
 7094 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 7095 %{
 7096   single_instruction;
 7097   cr     : EX1(read);
 7098   dst    : EX2(write);
 7099   INS01  : ISS;
 7100   ALU    : EX2;
 7101 %}
 7102 
 7103 // Conditional 2 operand
 7104 // EG.  CSEL    X0, X1, X2, <cond>
 7105 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 7106 %{
 7107   single_instruction;
 7108   cr     : EX1(read);
 7109   src1   : EX1(read);
 7110   src2   : EX1(read);
 7111   dst    : EX2(write);
 7112   INS01  : ISS;
 7113   ALU    : EX2;
 7114 %}
 7115 
 7116 // Conditional 2 operand
 7117 // EG.  CSEL    X0, X1, X2, <cond>
 7118 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 7119 %{
 7120   single_instruction;
 7121   cr     : EX1(read);
 7122   src    : EX1(read);
 7123   dst    : EX2(write);
 7124   INS01  : ISS;
 7125   ALU    : EX2;
 7126 %}
 7127 
 7128 //------- Multiply pipeline operations --------------------
 7129 
 7130 // Multiply reg-reg
 7131 // Eg.  MUL     w0, w1, w2
 7132 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7133 %{
 7134   single_instruction;
 7135   dst    : WR(write);
 7136   src1   : ISS(read);
 7137   src2   : ISS(read);
 7138   INS01  : ISS;
 7139   MAC    : WR;
 7140 %}
 7141 
 7142 // Multiply accumulate
 7143 // Eg.  MADD    w0, w1, w2, w3
 7144 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7145 %{
 7146   single_instruction;
 7147   dst    : WR(write);
 7148   src1   : ISS(read);
 7149   src2   : ISS(read);
 7150   src3   : ISS(read);
 7151   INS01  : ISS;
 7152   MAC    : WR;
 7153 %}
 7154 
 7155 // Eg.  MUL     w0, w1, w2
 7156 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7157 %{
 7158   single_instruction;
 7159   fixed_latency(3); // Maximum latency for 64 bit mul
 7160   dst    : WR(write);
 7161   src1   : ISS(read);
 7162   src2   : ISS(read);
 7163   INS01  : ISS;
 7164   MAC    : WR;
 7165 %}
 7166 
 7167 // Multiply accumulate
 7168 // Eg.  MADD    w0, w1, w2, w3
 7169 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7170 %{
 7171   single_instruction;
 7172   fixed_latency(3); // Maximum latency for 64 bit mul
 7173   dst    : WR(write);
 7174   src1   : ISS(read);
 7175   src2   : ISS(read);
 7176   src3   : ISS(read);
 7177   INS01  : ISS;
 7178   MAC    : WR;
 7179 %}
 7180 
 7181 //------- Divide pipeline operations --------------------
 7182 
 7183 // Eg.  SDIV    w0, w1, w2
 7184 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7185 %{
 7186   single_instruction;
 7187   fixed_latency(8); // Maximum latency for 32 bit divide
 7188   dst    : WR(write);
 7189   src1   : ISS(read);
 7190   src2   : ISS(read);
 7191   INS0   : ISS; // Can only dual issue as instruction 0
 7192   DIV    : WR;
 7193 %}
 7194 
 7195 // Eg.  SDIV    x0, x1, x2
 7196 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7197 %{
 7198   single_instruction;
 7199   fixed_latency(16); // Maximum latency for 64 bit divide
 7200   dst    : WR(write);
 7201   src1   : ISS(read);
 7202   src2   : ISS(read);
 7203   INS0   : ISS; // Can only dual issue as instruction 0
 7204   DIV    : WR;
 7205 %}
 7206 
 7207 //------- Load pipeline operations ------------------------
 7208 
 7209 // Load - prefetch
 7210 // Eg.  PFRM    <mem>
 7211 pipe_class iload_prefetch(memory mem)
 7212 %{
 7213   single_instruction;
 7214   mem    : ISS(read);
 7215   INS01  : ISS;
 7216   LDST   : WR;
 7217 %}
 7218 
 7219 // Load - reg, mem
 7220 // Eg.  LDR     x0, <mem>
 7221 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7222 %{
 7223   single_instruction;
 7224   dst    : WR(write);
 7225   mem    : ISS(read);
 7226   INS01  : ISS;
 7227   LDST   : WR;
 7228 %}
 7229 
 7230 // Load - reg, reg
 7231 // Eg.  LDR     x0, [sp, x1]
 7232 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7233 %{
 7234   single_instruction;
 7235   dst    : WR(write);
 7236   src    : ISS(read);
 7237   INS01  : ISS;
 7238   LDST   : WR;
 7239 %}
 7240 
 7241 //------- Store pipeline operations -----------------------
 7242 
 7243 // Store - zr, mem
 7244 // Eg.  STR     zr, <mem>
 7245 pipe_class istore_mem(memory mem)
 7246 %{
 7247   single_instruction;
 7248   mem    : ISS(read);
 7249   INS01  : ISS;
 7250   LDST   : WR;
 7251 %}
 7252 
 7253 // Store - reg, mem
 7254 // Eg.  STR     x0, <mem>
 7255 pipe_class istore_reg_mem(iRegI src, memory mem)
 7256 %{
 7257   single_instruction;
 7258   mem    : ISS(read);
 7259   src    : EX2(read);
 7260   INS01  : ISS;
 7261   LDST   : WR;
 7262 %}
 7263 
 7264 // Store - reg, reg
 7265 // Eg. STR      x0, [sp, x1]
 7266 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7267 %{
 7268   single_instruction;
 7269   dst    : ISS(read);
 7270   src    : EX2(read);
 7271   INS01  : ISS;
 7272   LDST   : WR;
 7273 %}
 7274 
 7275 //------- Store pipeline operations -----------------------
 7276 
 7277 // Branch
 7278 pipe_class pipe_branch()
 7279 %{
 7280   single_instruction;
 7281   INS01  : ISS;
 7282   BRANCH : EX1;
 7283 %}
 7284 
 7285 // Conditional branch
 7286 pipe_class pipe_branch_cond(rFlagsReg cr)
 7287 %{
 7288   single_instruction;
 7289   cr     : EX1(read);
 7290   INS01  : ISS;
 7291   BRANCH : EX1;
 7292 %}
 7293 
 7294 // Compare & Branch
 7295 // EG.  CBZ/CBNZ
 7296 pipe_class pipe_cmp_branch(iRegI op1)
 7297 %{
 7298   single_instruction;
 7299   op1    : EX1(read);
 7300   INS01  : ISS;
 7301   BRANCH : EX1;
 7302 %}
 7303 
 7304 //------- Synchronisation operations ----------------------
 7305 
 7306 // Any operation requiring serialization.
 7307 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7308 pipe_class pipe_serial()
 7309 %{
 7310   single_instruction;
 7311   force_serialization;
 7312   fixed_latency(16);
 7313   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7314   LDST   : WR;
 7315 %}
 7316 
 7317 // Generic big/slow expanded idiom - also serialized
 7318 pipe_class pipe_slow()
 7319 %{
 7320   instruction_count(10);
 7321   multiple_bundles;
 7322   force_serialization;
 7323   fixed_latency(16);
 7324   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7325   LDST   : WR;
 7326 %}
 7327 
 7328 // Empty pipeline class
 7329 pipe_class pipe_class_empty()
 7330 %{
 7331   single_instruction;
 7332   fixed_latency(0);
 7333 %}
 7334 
 7335 // Default pipeline class.
 7336 pipe_class pipe_class_default()
 7337 %{
 7338   single_instruction;
 7339   fixed_latency(2);
 7340 %}
 7341 
 7342 // Pipeline class for compares.
 7343 pipe_class pipe_class_compare()
 7344 %{
 7345   single_instruction;
 7346   fixed_latency(16);
 7347 %}
 7348 
 7349 // Pipeline class for memory operations.
 7350 pipe_class pipe_class_memory()
 7351 %{
 7352   single_instruction;
 7353   fixed_latency(16);
 7354 %}
 7355 
 7356 // Pipeline class for call.
 7357 pipe_class pipe_class_call()
 7358 %{
 7359   single_instruction;
 7360   fixed_latency(100);
 7361 %}
 7362 
 7363 // Define the class for the Nop node.
 7364 define %{
 7365    MachNop = pipe_class_empty;
 7366 %}
 7367 
 7368 %}
 7369 //----------INSTRUCTIONS-------------------------------------------------------
 7370 //
 7371 // match      -- States which machine-independent subtree may be replaced
 7372 //               by this instruction.
 7373 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7374 //               selection to identify a minimum cost tree of machine
 7375 //               instructions that matches a tree of machine-independent
 7376 //               instructions.
 7377 // format     -- A string providing the disassembly for this instruction.
 7378 //               The value of an instruction's operand may be inserted
 7379 //               by referring to it with a '$' prefix.
 7380 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7381 //               to within an encode class as $primary, $secondary, and $tertiary
 7382 //               rrspectively.  The primary opcode is commonly used to
 7383 //               indicate the type of machine instruction, while secondary
 7384 //               and tertiary are often used for prefix options or addressing
 7385 //               modes.
 7386 // ins_encode -- A list of encode classes with parameters. The encode class
 7387 //               name must have been defined in an 'enc_class' specification
 7388 //               in the encode section of the architecture description.
 7389 
 7390 // ============================================================================
 7391 // Memory (Load/Store) Instructions
 7392 
 7393 // Load Instructions
 7394 
 7395 // Load Byte (8 bit signed)
 7396 instruct loadB(iRegINoSp dst, memory1 mem)
 7397 %{
 7398   match(Set dst (LoadB mem));
 7399   predicate(!needs_acquiring_load(n));
 7400 
 7401   ins_cost(4 * INSN_COST);
 7402   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7403 
 7404   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7405 
 7406   ins_pipe(iload_reg_mem);
 7407 %}
 7408 
 7409 // Load Byte (8 bit signed) into long
 7410 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7411 %{
 7412   match(Set dst (ConvI2L (LoadB mem)));
 7413   predicate(!needs_acquiring_load(n->in(1)));
 7414 
 7415   ins_cost(4 * INSN_COST);
 7416   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7417 
 7418   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7419 
 7420   ins_pipe(iload_reg_mem);
 7421 %}
 7422 
 7423 // Load Byte (8 bit unsigned)
 7424 instruct loadUB(iRegINoSp dst, memory1 mem)
 7425 %{
 7426   match(Set dst (LoadUB mem));
 7427   predicate(!needs_acquiring_load(n));
 7428 
 7429   ins_cost(4 * INSN_COST);
 7430   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7431 
 7432   ins_encode(aarch64_enc_ldrb(dst, mem));
 7433 
 7434   ins_pipe(iload_reg_mem);
 7435 %}
 7436 
 7437 // Load Byte (8 bit unsigned) into long
 7438 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7439 %{
 7440   match(Set dst (ConvI2L (LoadUB mem)));
 7441   predicate(!needs_acquiring_load(n->in(1)));
 7442 
 7443   ins_cost(4 * INSN_COST);
 7444   format %{ "ldrb  $dst, $mem\t# byte" %}
 7445 
 7446   ins_encode(aarch64_enc_ldrb(dst, mem));
 7447 
 7448   ins_pipe(iload_reg_mem);
 7449 %}
 7450 
 7451 // Load Short (16 bit signed)
 7452 instruct loadS(iRegINoSp dst, memory2 mem)
 7453 %{
 7454   match(Set dst (LoadS mem));
 7455   predicate(!needs_acquiring_load(n));
 7456 
 7457   ins_cost(4 * INSN_COST);
 7458   format %{ "ldrshw  $dst, $mem\t# short" %}
 7459 
 7460   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7461 
 7462   ins_pipe(iload_reg_mem);
 7463 %}
 7464 
 7465 // Load Short (16 bit signed) into long
 7466 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7467 %{
 7468   match(Set dst (ConvI2L (LoadS mem)));
 7469   predicate(!needs_acquiring_load(n->in(1)));
 7470 
 7471   ins_cost(4 * INSN_COST);
 7472   format %{ "ldrsh  $dst, $mem\t# short" %}
 7473 
 7474   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7475 
 7476   ins_pipe(iload_reg_mem);
 7477 %}
 7478 
 7479 // Load Char (16 bit unsigned)
 7480 instruct loadUS(iRegINoSp dst, memory2 mem)
 7481 %{
 7482   match(Set dst (LoadUS mem));
 7483   predicate(!needs_acquiring_load(n));
 7484 
 7485   ins_cost(4 * INSN_COST);
 7486   format %{ "ldrh  $dst, $mem\t# short" %}
 7487 
 7488   ins_encode(aarch64_enc_ldrh(dst, mem));
 7489 
 7490   ins_pipe(iload_reg_mem);
 7491 %}
 7492 
 7493 // Load Short/Char (16 bit unsigned) into long
 7494 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7495 %{
 7496   match(Set dst (ConvI2L (LoadUS mem)));
 7497   predicate(!needs_acquiring_load(n->in(1)));
 7498 
 7499   ins_cost(4 * INSN_COST);
 7500   format %{ "ldrh  $dst, $mem\t# short" %}
 7501 
 7502   ins_encode(aarch64_enc_ldrh(dst, mem));
 7503 
 7504   ins_pipe(iload_reg_mem);
 7505 %}
 7506 
 7507 // Load Integer (32 bit signed)
 7508 instruct loadI(iRegINoSp dst, memory4 mem)
 7509 %{
 7510   match(Set dst (LoadI mem));
 7511   predicate(!needs_acquiring_load(n));
 7512 
 7513   ins_cost(4 * INSN_COST);
 7514   format %{ "ldrw  $dst, $mem\t# int" %}
 7515 
 7516   ins_encode(aarch64_enc_ldrw(dst, mem));
 7517 
 7518   ins_pipe(iload_reg_mem);
 7519 %}
 7520 
 7521 // Load Integer (32 bit signed) into long
 7522 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7523 %{
 7524   match(Set dst (ConvI2L (LoadI mem)));
 7525   predicate(!needs_acquiring_load(n->in(1)));
 7526 
 7527   ins_cost(4 * INSN_COST);
 7528   format %{ "ldrsw  $dst, $mem\t# int" %}
 7529 
 7530   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7531 
 7532   ins_pipe(iload_reg_mem);
 7533 %}
 7534 
 7535 // Load Integer (32 bit unsigned) into long
 7536 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7537 %{
 7538   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7539   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7540 
 7541   ins_cost(4 * INSN_COST);
 7542   format %{ "ldrw  $dst, $mem\t# int" %}
 7543 
 7544   ins_encode(aarch64_enc_ldrw(dst, mem));
 7545 
 7546   ins_pipe(iload_reg_mem);
 7547 %}
 7548 
 7549 // Load Long (64 bit signed)
 7550 instruct loadL(iRegLNoSp dst, memory8 mem)
 7551 %{
 7552   match(Set dst (LoadL mem));
 7553   predicate(!needs_acquiring_load(n));
 7554 
 7555   ins_cost(4 * INSN_COST);
 7556   format %{ "ldr  $dst, $mem\t# int" %}
 7557 
 7558   ins_encode(aarch64_enc_ldr(dst, mem));
 7559 
 7560   ins_pipe(iload_reg_mem);
 7561 %}
 7562 
 7563 // Load Range
 7564 instruct loadRange(iRegINoSp dst, memory4 mem)
 7565 %{
 7566   match(Set dst (LoadRange mem));
 7567 
 7568   ins_cost(4 * INSN_COST);
 7569   format %{ "ldrw  $dst, $mem\t# range" %}
 7570 
 7571   ins_encode(aarch64_enc_ldrw(dst, mem));
 7572 
 7573   ins_pipe(iload_reg_mem);
 7574 %}
 7575 
 7576 // Load Pointer
 7577 instruct loadP(iRegPNoSp dst, memory8 mem)
 7578 %{
 7579   match(Set dst (LoadP mem));
 7580   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7581 
 7582   ins_cost(4 * INSN_COST);
 7583   format %{ "ldr  $dst, $mem\t# ptr" %}
 7584 
 7585   ins_encode(aarch64_enc_ldr(dst, mem));
 7586 
 7587   ins_pipe(iload_reg_mem);
 7588 %}
 7589 
 7590 // Load Compressed Pointer
 7591 instruct loadN(iRegNNoSp dst, memory4 mem)
 7592 %{
 7593   match(Set dst (LoadN mem));
 7594   predicate(!needs_acquiring_load(n));
 7595 
 7596   ins_cost(4 * INSN_COST);
 7597   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7598 
 7599   ins_encode(aarch64_enc_ldrw(dst, mem));
 7600 
 7601   ins_pipe(iload_reg_mem);
 7602 %}
 7603 
 7604 // Load Klass Pointer
 7605 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7606 %{
 7607   match(Set dst (LoadKlass mem));
 7608   predicate(!needs_acquiring_load(n));
 7609 
 7610   ins_cost(4 * INSN_COST);
 7611   format %{ "ldr  $dst, $mem\t# class" %}
 7612 
 7613   ins_encode(aarch64_enc_ldr(dst, mem));
 7614 
 7615   ins_pipe(iload_reg_mem);
 7616 %}
 7617 
 7618 // Load Narrow Klass Pointer
 7619 instruct loadNKlass(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7620 %{
 7621   match(Set dst (LoadNKlass mem));
 7622   effect(TEMP_DEF dst, KILL cr);
 7623   predicate(!needs_acquiring_load(n));
 7624 
 7625   ins_cost(4 * INSN_COST);
 7626   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7627   ins_encode %{
 7628     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
 7629     assert($mem$$index$$Register == noreg, "expect no index");
 7630     __ load_nklass($dst$$Register, $mem$$base$$Register);
 7631   %}
 7632   ins_pipe(pipe_slow);
 7633 %}
 7634 
 7635 // Load Float
 7636 instruct loadF(vRegF dst, memory4 mem)
 7637 %{
 7638   match(Set dst (LoadF mem));
 7639   predicate(!needs_acquiring_load(n));
 7640 
 7641   ins_cost(4 * INSN_COST);
 7642   format %{ "ldrs  $dst, $mem\t# float" %}
 7643 
 7644   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7645 
 7646   ins_pipe(pipe_class_memory);
 7647 %}
 7648 
 7649 // Load Double
 7650 instruct loadD(vRegD dst, memory8 mem)
 7651 %{
 7652   match(Set dst (LoadD mem));
 7653   predicate(!needs_acquiring_load(n));
 7654 
 7655   ins_cost(4 * INSN_COST);
 7656   format %{ "ldrd  $dst, $mem\t# double" %}
 7657 
 7658   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7659 
 7660   ins_pipe(pipe_class_memory);
 7661 %}
 7662 
 7663 
 7664 // Load Int Constant
 7665 instruct loadConI(iRegINoSp dst, immI src)
 7666 %{
 7667   match(Set dst src);
 7668 
 7669   ins_cost(INSN_COST);
 7670   format %{ "mov $dst, $src\t# int" %}
 7671 
 7672   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7673 
 7674   ins_pipe(ialu_imm);
 7675 %}
 7676 
 7677 // Load Long Constant
 7678 instruct loadConL(iRegLNoSp dst, immL src)
 7679 %{
 7680   match(Set dst src);
 7681 
 7682   ins_cost(INSN_COST);
 7683   format %{ "mov $dst, $src\t# long" %}
 7684 
 7685   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7686 
 7687   ins_pipe(ialu_imm);
 7688 %}
 7689 
 7690 // Load Pointer Constant
 7691 
 7692 instruct loadConP(iRegPNoSp dst, immP con)
 7693 %{
 7694   match(Set dst con);
 7695 
 7696   ins_cost(INSN_COST * 4);
 7697   format %{
 7698     "mov  $dst, $con\t# ptr\n\t"
 7699   %}
 7700 
 7701   ins_encode(aarch64_enc_mov_p(dst, con));
 7702 
 7703   ins_pipe(ialu_imm);
 7704 %}
 7705 
 7706 // Load Null Pointer Constant
 7707 
 7708 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7709 %{
 7710   match(Set dst con);
 7711 
 7712   ins_cost(INSN_COST);
 7713   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7714 
 7715   ins_encode(aarch64_enc_mov_p0(dst, con));
 7716 
 7717   ins_pipe(ialu_imm);
 7718 %}
 7719 
 7720 // Load Pointer Constant One
 7721 
 7722 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7723 %{
 7724   match(Set dst con);
 7725 
 7726   ins_cost(INSN_COST);
 7727   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7728 
 7729   ins_encode(aarch64_enc_mov_p1(dst, con));
 7730 
 7731   ins_pipe(ialu_imm);
 7732 %}
 7733 
 7734 // Load Byte Map Base Constant
 7735 
 7736 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7737 %{
 7738   match(Set dst con);
 7739 
 7740   ins_cost(INSN_COST);
 7741   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7742 
 7743   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7744 
 7745   ins_pipe(ialu_imm);
 7746 %}
 7747 
 7748 // Load Narrow Pointer Constant
 7749 
 7750 instruct loadConN(iRegNNoSp dst, immN con)
 7751 %{
 7752   match(Set dst con);
 7753 
 7754   ins_cost(INSN_COST * 4);
 7755   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7756 
 7757   ins_encode(aarch64_enc_mov_n(dst, con));
 7758 
 7759   ins_pipe(ialu_imm);
 7760 %}
 7761 
 7762 // Load Narrow Null Pointer Constant
 7763 
 7764 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7765 %{
 7766   match(Set dst con);
 7767 
 7768   ins_cost(INSN_COST);
 7769   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7770 
 7771   ins_encode(aarch64_enc_mov_n0(dst, con));
 7772 
 7773   ins_pipe(ialu_imm);
 7774 %}
 7775 
 7776 // Load Narrow Klass Constant
 7777 
 7778 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7779 %{
 7780   match(Set dst con);
 7781 
 7782   ins_cost(INSN_COST);
 7783   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7784 
 7785   ins_encode(aarch64_enc_mov_nk(dst, con));
 7786 
 7787   ins_pipe(ialu_imm);
 7788 %}
 7789 
 7790 // Load Packed Float Constant
 7791 
 7792 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7793   match(Set dst con);
 7794   ins_cost(INSN_COST * 4);
 7795   format %{ "fmovs  $dst, $con"%}
 7796   ins_encode %{
 7797     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7798   %}
 7799 
 7800   ins_pipe(fp_imm_s);
 7801 %}
 7802 
 7803 // Load Float Constant
 7804 
 7805 instruct loadConF(vRegF dst, immF con) %{
 7806   match(Set dst con);
 7807 
 7808   ins_cost(INSN_COST * 4);
 7809 
 7810   format %{
 7811     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7812   %}
 7813 
 7814   ins_encode %{
 7815     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7816   %}
 7817 
 7818   ins_pipe(fp_load_constant_s);
 7819 %}
 7820 
 7821 // Load Packed Double Constant
 7822 
 7823 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7824   match(Set dst con);
 7825   ins_cost(INSN_COST);
 7826   format %{ "fmovd  $dst, $con"%}
 7827   ins_encode %{
 7828     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7829   %}
 7830 
 7831   ins_pipe(fp_imm_d);
 7832 %}
 7833 
 7834 // Load Double Constant
 7835 
 7836 instruct loadConD(vRegD dst, immD con) %{
 7837   match(Set dst con);
 7838 
 7839   ins_cost(INSN_COST * 5);
 7840   format %{
 7841     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7842   %}
 7843 
 7844   ins_encode %{
 7845     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7846   %}
 7847 
 7848   ins_pipe(fp_load_constant_d);
 7849 %}
 7850 
 7851 // Store Instructions
 7852 
 7853 // Store CMS card-mark Immediate
 7854 instruct storeimmCM0(immI0 zero, memory1 mem)
 7855 %{
 7856   match(Set mem (StoreCM mem zero));
 7857 
 7858   ins_cost(INSN_COST);
 7859   format %{ "storestore (elided)\n\t"
 7860             "strb zr, $mem\t# byte" %}
 7861 
 7862   ins_encode(aarch64_enc_strb0(mem));
 7863 
 7864   ins_pipe(istore_mem);
 7865 %}
 7866 
 7867 // Store CMS card-mark Immediate with intervening StoreStore
 7868 // needed when using CMS with no conditional card marking
 7869 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7870 %{
 7871   match(Set mem (StoreCM mem zero));
 7872 
 7873   ins_cost(INSN_COST * 2);
 7874   format %{ "storestore\n\t"
 7875             "dmb ishst"
 7876             "\n\tstrb zr, $mem\t# byte" %}
 7877 
 7878   ins_encode(aarch64_enc_strb0_ordered(mem));
 7879 
 7880   ins_pipe(istore_mem);
 7881 %}
 7882 
 7883 // Store Byte
 7884 instruct storeB(iRegIorL2I src, memory1 mem)
 7885 %{
 7886   match(Set mem (StoreB mem src));
 7887   predicate(!needs_releasing_store(n));
 7888 
 7889   ins_cost(INSN_COST);
 7890   format %{ "strb  $src, $mem\t# byte" %}
 7891 
 7892   ins_encode(aarch64_enc_strb(src, mem));
 7893 
 7894   ins_pipe(istore_reg_mem);
 7895 %}
 7896 
 7897 
 7898 instruct storeimmB0(immI0 zero, memory1 mem)
 7899 %{
 7900   match(Set mem (StoreB mem zero));
 7901   predicate(!needs_releasing_store(n));
 7902 
 7903   ins_cost(INSN_COST);
 7904   format %{ "strb rscractch2, $mem\t# byte" %}
 7905 
 7906   ins_encode(aarch64_enc_strb0(mem));
 7907 
 7908   ins_pipe(istore_mem);
 7909 %}
 7910 
 7911 // Store Char/Short
 7912 instruct storeC(iRegIorL2I src, memory2 mem)
 7913 %{
 7914   match(Set mem (StoreC mem src));
 7915   predicate(!needs_releasing_store(n));
 7916 
 7917   ins_cost(INSN_COST);
 7918   format %{ "strh  $src, $mem\t# short" %}
 7919 
 7920   ins_encode(aarch64_enc_strh(src, mem));
 7921 
 7922   ins_pipe(istore_reg_mem);
 7923 %}
 7924 
 7925 instruct storeimmC0(immI0 zero, memory2 mem)
 7926 %{
 7927   match(Set mem (StoreC mem zero));
 7928   predicate(!needs_releasing_store(n));
 7929 
 7930   ins_cost(INSN_COST);
 7931   format %{ "strh  zr, $mem\t# short" %}
 7932 
 7933   ins_encode(aarch64_enc_strh0(mem));
 7934 
 7935   ins_pipe(istore_mem);
 7936 %}
 7937 
 7938 // Store Integer
 7939 
 7940 instruct storeI(iRegIorL2I src, memory4 mem)
 7941 %{
 7942   match(Set mem(StoreI mem src));
 7943   predicate(!needs_releasing_store(n));
 7944 
 7945   ins_cost(INSN_COST);
 7946   format %{ "strw  $src, $mem\t# int" %}
 7947 
 7948   ins_encode(aarch64_enc_strw(src, mem));
 7949 
 7950   ins_pipe(istore_reg_mem);
 7951 %}
 7952 
 7953 instruct storeimmI0(immI0 zero, memory4 mem)
 7954 %{
 7955   match(Set mem(StoreI mem zero));
 7956   predicate(!needs_releasing_store(n));
 7957 
 7958   ins_cost(INSN_COST);
 7959   format %{ "strw  zr, $mem\t# int" %}
 7960 
 7961   ins_encode(aarch64_enc_strw0(mem));
 7962 
 7963   ins_pipe(istore_mem);
 7964 %}
 7965 
 7966 // Store Long (64 bit signed)
 7967 instruct storeL(iRegL src, memory8 mem)
 7968 %{
 7969   match(Set mem (StoreL mem src));
 7970   predicate(!needs_releasing_store(n));
 7971 
 7972   ins_cost(INSN_COST);
 7973   format %{ "str  $src, $mem\t# int" %}
 7974 
 7975   ins_encode(aarch64_enc_str(src, mem));
 7976 
 7977   ins_pipe(istore_reg_mem);
 7978 %}
 7979 
 7980 // Store Long (64 bit signed)
 7981 instruct storeimmL0(immL0 zero, memory8 mem)
 7982 %{
 7983   match(Set mem (StoreL mem zero));
 7984   predicate(!needs_releasing_store(n));
 7985 
 7986   ins_cost(INSN_COST);
 7987   format %{ "str  zr, $mem\t# int" %}
 7988 
 7989   ins_encode(aarch64_enc_str0(mem));
 7990 
 7991   ins_pipe(istore_mem);
 7992 %}
 7993 
 7994 // Store Pointer
 7995 instruct storeP(iRegP src, memory8 mem)
 7996 %{
 7997   match(Set mem (StoreP mem src));
 7998   predicate(!needs_releasing_store(n));
 7999 
 8000   ins_cost(INSN_COST);
 8001   format %{ "str  $src, $mem\t# ptr" %}
 8002 
 8003   ins_encode(aarch64_enc_str(src, mem));
 8004 
 8005   ins_pipe(istore_reg_mem);
 8006 %}
 8007 
 8008 // Store Pointer
 8009 instruct storeimmP0(immP0 zero, memory8 mem)
 8010 %{
 8011   match(Set mem (StoreP mem zero));
 8012   predicate(!needs_releasing_store(n));
 8013 
 8014   ins_cost(INSN_COST);
 8015   format %{ "str zr, $mem\t# ptr" %}
 8016 
 8017   ins_encode(aarch64_enc_str0(mem));
 8018 
 8019   ins_pipe(istore_mem);
 8020 %}
 8021 
 8022 // Store Compressed Pointer
 8023 instruct storeN(iRegN src, memory4 mem)
 8024 %{
 8025   match(Set mem (StoreN mem src));
 8026   predicate(!needs_releasing_store(n));
 8027 
 8028   ins_cost(INSN_COST);
 8029   format %{ "strw  $src, $mem\t# compressed ptr" %}
 8030 
 8031   ins_encode(aarch64_enc_strw(src, mem));
 8032 
 8033   ins_pipe(istore_reg_mem);
 8034 %}
 8035 
 8036 instruct storeImmN0(immN0 zero, memory4 mem)
 8037 %{
 8038   match(Set mem (StoreN mem zero));
 8039   predicate(!needs_releasing_store(n));
 8040 
 8041   ins_cost(INSN_COST);
 8042   format %{ "strw  zr, $mem\t# compressed ptr" %}
 8043 
 8044   ins_encode(aarch64_enc_strw0(mem));
 8045 
 8046   ins_pipe(istore_mem);
 8047 %}
 8048 
 8049 // Store Float
 8050 instruct storeF(vRegF src, memory4 mem)
 8051 %{
 8052   match(Set mem (StoreF mem src));
 8053   predicate(!needs_releasing_store(n));
 8054 
 8055   ins_cost(INSN_COST);
 8056   format %{ "strs  $src, $mem\t# float" %}
 8057 
 8058   ins_encode( aarch64_enc_strs(src, mem) );
 8059 
 8060   ins_pipe(pipe_class_memory);
 8061 %}
 8062 
 8063 // TODO
 8064 // implement storeImmF0 and storeFImmPacked
 8065 
 8066 // Store Double
 8067 instruct storeD(vRegD src, memory8 mem)
 8068 %{
 8069   match(Set mem (StoreD mem src));
 8070   predicate(!needs_releasing_store(n));
 8071 
 8072   ins_cost(INSN_COST);
 8073   format %{ "strd  $src, $mem\t# double" %}
 8074 
 8075   ins_encode( aarch64_enc_strd(src, mem) );
 8076 
 8077   ins_pipe(pipe_class_memory);
 8078 %}
 8079 
 8080 // Store Compressed Klass Pointer
 8081 instruct storeNKlass(iRegN src, memory4 mem)
 8082 %{
 8083   predicate(!needs_releasing_store(n));
 8084   match(Set mem (StoreNKlass mem src));
 8085 
 8086   ins_cost(INSN_COST);
 8087   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 8088 
 8089   ins_encode(aarch64_enc_strw(src, mem));
 8090 
 8091   ins_pipe(istore_reg_mem);
 8092 %}
 8093 
 8094 // TODO
 8095 // implement storeImmD0 and storeDImmPacked
 8096 
 8097 // prefetch instructions
 8098 // Must be safe to execute with invalid address (cannot fault).
 8099 
 8100 instruct prefetchalloc( memory8 mem ) %{
 8101   match(PrefetchAllocation mem);
 8102 
 8103   ins_cost(INSN_COST);
 8104   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 8105 
 8106   ins_encode( aarch64_enc_prefetchw(mem) );
 8107 
 8108   ins_pipe(iload_prefetch);
 8109 %}
 8110 
 8111 //  ---------------- volatile loads and stores ----------------
 8112 
 8113 // Load Byte (8 bit signed)
 8114 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8115 %{
 8116   match(Set dst (LoadB mem));
 8117 
 8118   ins_cost(VOLATILE_REF_COST);
 8119   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8120 
 8121   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8122 
 8123   ins_pipe(pipe_serial);
 8124 %}
 8125 
 8126 // Load Byte (8 bit signed) into long
 8127 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8128 %{
 8129   match(Set dst (ConvI2L (LoadB mem)));
 8130 
 8131   ins_cost(VOLATILE_REF_COST);
 8132   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8133 
 8134   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8135 
 8136   ins_pipe(pipe_serial);
 8137 %}
 8138 
 8139 // Load Byte (8 bit unsigned)
 8140 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8141 %{
 8142   match(Set dst (LoadUB mem));
 8143 
 8144   ins_cost(VOLATILE_REF_COST);
 8145   format %{ "ldarb  $dst, $mem\t# byte" %}
 8146 
 8147   ins_encode(aarch64_enc_ldarb(dst, mem));
 8148 
 8149   ins_pipe(pipe_serial);
 8150 %}
 8151 
 8152 // Load Byte (8 bit unsigned) into long
 8153 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8154 %{
 8155   match(Set dst (ConvI2L (LoadUB mem)));
 8156 
 8157   ins_cost(VOLATILE_REF_COST);
 8158   format %{ "ldarb  $dst, $mem\t# byte" %}
 8159 
 8160   ins_encode(aarch64_enc_ldarb(dst, mem));
 8161 
 8162   ins_pipe(pipe_serial);
 8163 %}
 8164 
 8165 // Load Short (16 bit signed)
 8166 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8167 %{
 8168   match(Set dst (LoadS mem));
 8169 
 8170   ins_cost(VOLATILE_REF_COST);
 8171   format %{ "ldarshw  $dst, $mem\t# short" %}
 8172 
 8173   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8174 
 8175   ins_pipe(pipe_serial);
 8176 %}
 8177 
 8178 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8179 %{
 8180   match(Set dst (LoadUS mem));
 8181 
 8182   ins_cost(VOLATILE_REF_COST);
 8183   format %{ "ldarhw  $dst, $mem\t# short" %}
 8184 
 8185   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8186 
 8187   ins_pipe(pipe_serial);
 8188 %}
 8189 
 8190 // Load Short/Char (16 bit unsigned) into long
 8191 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8192 %{
 8193   match(Set dst (ConvI2L (LoadUS mem)));
 8194 
 8195   ins_cost(VOLATILE_REF_COST);
 8196   format %{ "ldarh  $dst, $mem\t# short" %}
 8197 
 8198   ins_encode(aarch64_enc_ldarh(dst, mem));
 8199 
 8200   ins_pipe(pipe_serial);
 8201 %}
 8202 
 8203 // Load Short/Char (16 bit signed) into long
 8204 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8205 %{
 8206   match(Set dst (ConvI2L (LoadS mem)));
 8207 
 8208   ins_cost(VOLATILE_REF_COST);
 8209   format %{ "ldarh  $dst, $mem\t# short" %}
 8210 
 8211   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8212 
 8213   ins_pipe(pipe_serial);
 8214 %}
 8215 
 8216 // Load Integer (32 bit signed)
 8217 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8218 %{
 8219   match(Set dst (LoadI mem));
 8220 
 8221   ins_cost(VOLATILE_REF_COST);
 8222   format %{ "ldarw  $dst, $mem\t# int" %}
 8223 
 8224   ins_encode(aarch64_enc_ldarw(dst, mem));
 8225 
 8226   ins_pipe(pipe_serial);
 8227 %}
 8228 
 8229 // Load Integer (32 bit unsigned) into long
 8230 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8231 %{
 8232   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8233 
 8234   ins_cost(VOLATILE_REF_COST);
 8235   format %{ "ldarw  $dst, $mem\t# int" %}
 8236 
 8237   ins_encode(aarch64_enc_ldarw(dst, mem));
 8238 
 8239   ins_pipe(pipe_serial);
 8240 %}
 8241 
 8242 // Load Long (64 bit signed)
 8243 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8244 %{
 8245   match(Set dst (LoadL mem));
 8246 
 8247   ins_cost(VOLATILE_REF_COST);
 8248   format %{ "ldar  $dst, $mem\t# int" %}
 8249 
 8250   ins_encode(aarch64_enc_ldar(dst, mem));
 8251 
 8252   ins_pipe(pipe_serial);
 8253 %}
 8254 
 8255 // Load Pointer
 8256 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8257 %{
 8258   match(Set dst (LoadP mem));
 8259   predicate(n->as_Load()->barrier_data() == 0);
 8260 
 8261   ins_cost(VOLATILE_REF_COST);
 8262   format %{ "ldar  $dst, $mem\t# ptr" %}
 8263 
 8264   ins_encode(aarch64_enc_ldar(dst, mem));
 8265 
 8266   ins_pipe(pipe_serial);
 8267 %}
 8268 
 8269 // Load Compressed Pointer
 8270 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8271 %{
 8272   match(Set dst (LoadN mem));
 8273 
 8274   ins_cost(VOLATILE_REF_COST);
 8275   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8276 
 8277   ins_encode(aarch64_enc_ldarw(dst, mem));
 8278 
 8279   ins_pipe(pipe_serial);
 8280 %}
 8281 
 8282 // Load Float
 8283 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8284 %{
 8285   match(Set dst (LoadF mem));
 8286 
 8287   ins_cost(VOLATILE_REF_COST);
 8288   format %{ "ldars  $dst, $mem\t# float" %}
 8289 
 8290   ins_encode( aarch64_enc_fldars(dst, mem) );
 8291 
 8292   ins_pipe(pipe_serial);
 8293 %}
 8294 
 8295 // Load Double
 8296 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8297 %{
 8298   match(Set dst (LoadD mem));
 8299 
 8300   ins_cost(VOLATILE_REF_COST);
 8301   format %{ "ldard  $dst, $mem\t# double" %}
 8302 
 8303   ins_encode( aarch64_enc_fldard(dst, mem) );
 8304 
 8305   ins_pipe(pipe_serial);
 8306 %}
 8307 
 8308 // Store Byte
 8309 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8310 %{
 8311   match(Set mem (StoreB mem src));
 8312 
 8313   ins_cost(VOLATILE_REF_COST);
 8314   format %{ "stlrb  $src, $mem\t# byte" %}
 8315 
 8316   ins_encode(aarch64_enc_stlrb(src, mem));
 8317 
 8318   ins_pipe(pipe_class_memory);
 8319 %}
 8320 
 8321 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8322 %{
 8323   match(Set mem (StoreB mem zero));
 8324 
 8325   ins_cost(VOLATILE_REF_COST);
 8326   format %{ "stlrb  zr, $mem\t# byte" %}
 8327 
 8328   ins_encode(aarch64_enc_stlrb0(mem));
 8329 
 8330   ins_pipe(pipe_class_memory);
 8331 %}
 8332 
 8333 // Store Char/Short
 8334 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8335 %{
 8336   match(Set mem (StoreC mem src));
 8337 
 8338   ins_cost(VOLATILE_REF_COST);
 8339   format %{ "stlrh  $src, $mem\t# short" %}
 8340 
 8341   ins_encode(aarch64_enc_stlrh(src, mem));
 8342 
 8343   ins_pipe(pipe_class_memory);
 8344 %}
 8345 
 8346 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8347 %{
 8348   match(Set mem (StoreC mem zero));
 8349 
 8350   ins_cost(VOLATILE_REF_COST);
 8351   format %{ "stlrh  zr, $mem\t# short" %}
 8352 
 8353   ins_encode(aarch64_enc_stlrh0(mem));
 8354 
 8355   ins_pipe(pipe_class_memory);
 8356 %}
 8357 
 8358 // Store Integer
 8359 
 8360 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8361 %{
 8362   match(Set mem(StoreI mem src));
 8363 
 8364   ins_cost(VOLATILE_REF_COST);
 8365   format %{ "stlrw  $src, $mem\t# int" %}
 8366 
 8367   ins_encode(aarch64_enc_stlrw(src, mem));
 8368 
 8369   ins_pipe(pipe_class_memory);
 8370 %}
 8371 
 8372 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8373 %{
 8374   match(Set mem(StoreI mem zero));
 8375 
 8376   ins_cost(VOLATILE_REF_COST);
 8377   format %{ "stlrw  zr, $mem\t# int" %}
 8378 
 8379   ins_encode(aarch64_enc_stlrw0(mem));
 8380 
 8381   ins_pipe(pipe_class_memory);
 8382 %}
 8383 
 8384 // Store Long (64 bit signed)
 8385 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8386 %{
 8387   match(Set mem (StoreL mem src));
 8388 
 8389   ins_cost(VOLATILE_REF_COST);
 8390   format %{ "stlr  $src, $mem\t# int" %}
 8391 
 8392   ins_encode(aarch64_enc_stlr(src, mem));
 8393 
 8394   ins_pipe(pipe_class_memory);
 8395 %}
 8396 
 8397 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8398 %{
 8399   match(Set mem (StoreL mem zero));
 8400 
 8401   ins_cost(VOLATILE_REF_COST);
 8402   format %{ "stlr  zr, $mem\t# int" %}
 8403 
 8404   ins_encode(aarch64_enc_stlr0(mem));
 8405 
 8406   ins_pipe(pipe_class_memory);
 8407 %}
 8408 
 8409 // Store Pointer
 8410 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8411 %{
 8412   match(Set mem (StoreP mem src));
 8413 
 8414   ins_cost(VOLATILE_REF_COST);
 8415   format %{ "stlr  $src, $mem\t# ptr" %}
 8416 
 8417   ins_encode(aarch64_enc_stlr(src, mem));
 8418 
 8419   ins_pipe(pipe_class_memory);
 8420 %}
 8421 
 8422 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8423 %{
 8424   match(Set mem (StoreP mem zero));
 8425 
 8426   ins_cost(VOLATILE_REF_COST);
 8427   format %{ "stlr  zr, $mem\t# ptr" %}
 8428 
 8429   ins_encode(aarch64_enc_stlr0(mem));
 8430 
 8431   ins_pipe(pipe_class_memory);
 8432 %}
 8433 
 8434 // Store Compressed Pointer
 8435 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8436 %{
 8437   match(Set mem (StoreN mem src));
 8438 
 8439   ins_cost(VOLATILE_REF_COST);
 8440   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8441 
 8442   ins_encode(aarch64_enc_stlrw(src, mem));
 8443 
 8444   ins_pipe(pipe_class_memory);
 8445 %}
 8446 
 8447 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8448 %{
 8449   match(Set mem (StoreN mem zero));
 8450 
 8451   ins_cost(VOLATILE_REF_COST);
 8452   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8453 
 8454   ins_encode(aarch64_enc_stlrw0(mem));
 8455 
 8456   ins_pipe(pipe_class_memory);
 8457 %}
 8458 
 8459 // Store Float
 8460 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8461 %{
 8462   match(Set mem (StoreF mem src));
 8463 
 8464   ins_cost(VOLATILE_REF_COST);
 8465   format %{ "stlrs  $src, $mem\t# float" %}
 8466 
 8467   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8468 
 8469   ins_pipe(pipe_class_memory);
 8470 %}
 8471 
 8472 // TODO
 8473 // implement storeImmF0 and storeFImmPacked
 8474 
 8475 // Store Double
 8476 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8477 %{
 8478   match(Set mem (StoreD mem src));
 8479 
 8480   ins_cost(VOLATILE_REF_COST);
 8481   format %{ "stlrd  $src, $mem\t# double" %}
 8482 
 8483   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8484 
 8485   ins_pipe(pipe_class_memory);
 8486 %}
 8487 
 8488 //  ---------------- end of volatile loads and stores ----------------
 8489 
 8490 instruct cacheWB(indirect addr)
 8491 %{
 8492   predicate(VM_Version::supports_data_cache_line_flush());
 8493   match(CacheWB addr);
 8494 
 8495   ins_cost(100);
 8496   format %{"cache wb $addr" %}
 8497   ins_encode %{
 8498     assert($addr->index_position() < 0, "should be");
 8499     assert($addr$$disp == 0, "should be");
 8500     __ cache_wb(Address($addr$$base$$Register, 0));
 8501   %}
 8502   ins_pipe(pipe_slow); // XXX
 8503 %}
 8504 
 8505 instruct cacheWBPreSync()
 8506 %{
 8507   predicate(VM_Version::supports_data_cache_line_flush());
 8508   match(CacheWBPreSync);
 8509 
 8510   ins_cost(100);
 8511   format %{"cache wb presync" %}
 8512   ins_encode %{
 8513     __ cache_wbsync(true);
 8514   %}
 8515   ins_pipe(pipe_slow); // XXX
 8516 %}
 8517 
 8518 instruct cacheWBPostSync()
 8519 %{
 8520   predicate(VM_Version::supports_data_cache_line_flush());
 8521   match(CacheWBPostSync);
 8522 
 8523   ins_cost(100);
 8524   format %{"cache wb postsync" %}
 8525   ins_encode %{
 8526     __ cache_wbsync(false);
 8527   %}
 8528   ins_pipe(pipe_slow); // XXX
 8529 %}
 8530 
 8531 // ============================================================================
 8532 // BSWAP Instructions
 8533 
 8534 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8535   match(Set dst (ReverseBytesI src));
 8536 
 8537   ins_cost(INSN_COST);
 8538   format %{ "revw  $dst, $src" %}
 8539 
 8540   ins_encode %{
 8541     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8542   %}
 8543 
 8544   ins_pipe(ialu_reg);
 8545 %}
 8546 
 8547 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8548   match(Set dst (ReverseBytesL src));
 8549 
 8550   ins_cost(INSN_COST);
 8551   format %{ "rev  $dst, $src" %}
 8552 
 8553   ins_encode %{
 8554     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8555   %}
 8556 
 8557   ins_pipe(ialu_reg);
 8558 %}
 8559 
 8560 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8561   match(Set dst (ReverseBytesUS src));
 8562 
 8563   ins_cost(INSN_COST);
 8564   format %{ "rev16w  $dst, $src" %}
 8565 
 8566   ins_encode %{
 8567     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8568   %}
 8569 
 8570   ins_pipe(ialu_reg);
 8571 %}
 8572 
 8573 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8574   match(Set dst (ReverseBytesS src));
 8575 
 8576   ins_cost(INSN_COST);
 8577   format %{ "rev16w  $dst, $src\n\t"
 8578             "sbfmw $dst, $dst, #0, #15" %}
 8579 
 8580   ins_encode %{
 8581     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8582     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8583   %}
 8584 
 8585   ins_pipe(ialu_reg);
 8586 %}
 8587 
 8588 // ============================================================================
 8589 // Zero Count Instructions
 8590 
 8591 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8592   match(Set dst (CountLeadingZerosI src));
 8593 
 8594   ins_cost(INSN_COST);
 8595   format %{ "clzw  $dst, $src" %}
 8596   ins_encode %{
 8597     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8598   %}
 8599 
 8600   ins_pipe(ialu_reg);
 8601 %}
 8602 
 8603 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8604   match(Set dst (CountLeadingZerosL src));
 8605 
 8606   ins_cost(INSN_COST);
 8607   format %{ "clz   $dst, $src" %}
 8608   ins_encode %{
 8609     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8610   %}
 8611 
 8612   ins_pipe(ialu_reg);
 8613 %}
 8614 
 8615 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8616   match(Set dst (CountTrailingZerosI src));
 8617 
 8618   ins_cost(INSN_COST * 2);
 8619   format %{ "rbitw  $dst, $src\n\t"
 8620             "clzw   $dst, $dst" %}
 8621   ins_encode %{
 8622     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8623     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8624   %}
 8625 
 8626   ins_pipe(ialu_reg);
 8627 %}
 8628 
 8629 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8630   match(Set dst (CountTrailingZerosL src));
 8631 
 8632   ins_cost(INSN_COST * 2);
 8633   format %{ "rbit   $dst, $src\n\t"
 8634             "clz    $dst, $dst" %}
 8635   ins_encode %{
 8636     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8637     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8638   %}
 8639 
 8640   ins_pipe(ialu_reg);
 8641 %}
 8642 
 8643 //---------- Population Count Instructions -------------------------------------
 8644 //
 8645 
 8646 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8647   predicate(UsePopCountInstruction);
 8648   match(Set dst (PopCountI src));
 8649   effect(TEMP tmp);
 8650   ins_cost(INSN_COST * 13);
 8651 
 8652   format %{ "movw   $src, $src\n\t"
 8653             "mov    $tmp, $src\t# vector (1D)\n\t"
 8654             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8655             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8656             "mov    $dst, $tmp\t# vector (1D)" %}
 8657   ins_encode %{
 8658     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8659     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8660     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8661     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8662     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8663   %}
 8664 
 8665   ins_pipe(pipe_class_default);
 8666 %}
 8667 
 8668 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8669   predicate(UsePopCountInstruction);
 8670   match(Set dst (PopCountI (LoadI mem)));
 8671   effect(TEMP tmp);
 8672   ins_cost(INSN_COST * 13);
 8673 
 8674   format %{ "ldrs   $tmp, $mem\n\t"
 8675             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8676             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8677             "mov    $dst, $tmp\t# vector (1D)" %}
 8678   ins_encode %{
 8679     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8680     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8681               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8682     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8683     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8684     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8685   %}
 8686 
 8687   ins_pipe(pipe_class_default);
 8688 %}
 8689 
 8690 // Note: Long.bitCount(long) returns an int.
 8691 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8692   predicate(UsePopCountInstruction);
 8693   match(Set dst (PopCountL src));
 8694   effect(TEMP tmp);
 8695   ins_cost(INSN_COST * 13);
 8696 
 8697   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8698             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8699             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8700             "mov    $dst, $tmp\t# vector (1D)" %}
 8701   ins_encode %{
 8702     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8703     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8704     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8705     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8706   %}
 8707 
 8708   ins_pipe(pipe_class_default);
 8709 %}
 8710 
 8711 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8712   predicate(UsePopCountInstruction);
 8713   match(Set dst (PopCountL (LoadL mem)));
 8714   effect(TEMP tmp);
 8715   ins_cost(INSN_COST * 13);
 8716 
 8717   format %{ "ldrd   $tmp, $mem\n\t"
 8718             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8719             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8720             "mov    $dst, $tmp\t# vector (1D)" %}
 8721   ins_encode %{
 8722     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8723     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8724               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8725     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8726     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8727     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8728   %}
 8729 
 8730   ins_pipe(pipe_class_default);
 8731 %}
 8732 
 8733 // ============================================================================
 8734 // MemBar Instruction
 8735 
 8736 instruct load_fence() %{
 8737   match(LoadFence);
 8738   ins_cost(VOLATILE_REF_COST);
 8739 
 8740   format %{ "load_fence" %}
 8741 
 8742   ins_encode %{
 8743     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8744   %}
 8745   ins_pipe(pipe_serial);
 8746 %}
 8747 
 8748 instruct unnecessary_membar_acquire() %{
 8749   predicate(unnecessary_acquire(n));
 8750   match(MemBarAcquire);
 8751   ins_cost(0);
 8752 
 8753   format %{ "membar_acquire (elided)" %}
 8754 
 8755   ins_encode %{
 8756     __ block_comment("membar_acquire (elided)");
 8757   %}
 8758 
 8759   ins_pipe(pipe_class_empty);
 8760 %}
 8761 
 8762 instruct membar_acquire() %{
 8763   match(MemBarAcquire);
 8764   ins_cost(VOLATILE_REF_COST);
 8765 
 8766   format %{ "membar_acquire\n\t"
 8767             "dmb ish" %}
 8768 
 8769   ins_encode %{
 8770     __ block_comment("membar_acquire");
 8771     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8772   %}
 8773 
 8774   ins_pipe(pipe_serial);
 8775 %}
 8776 
 8777 
 8778 instruct membar_acquire_lock() %{
 8779   match(MemBarAcquireLock);
 8780   ins_cost(VOLATILE_REF_COST);
 8781 
 8782   format %{ "membar_acquire_lock (elided)" %}
 8783 
 8784   ins_encode %{
 8785     __ block_comment("membar_acquire_lock (elided)");
 8786   %}
 8787 
 8788   ins_pipe(pipe_serial);
 8789 %}
 8790 
 8791 instruct store_fence() %{
 8792   match(StoreFence);
 8793   ins_cost(VOLATILE_REF_COST);
 8794 
 8795   format %{ "store_fence" %}
 8796 
 8797   ins_encode %{
 8798     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8799   %}
 8800   ins_pipe(pipe_serial);
 8801 %}
 8802 
 8803 instruct unnecessary_membar_release() %{
 8804   predicate(unnecessary_release(n));
 8805   match(MemBarRelease);
 8806   ins_cost(0);
 8807 
 8808   format %{ "membar_release (elided)" %}
 8809 
 8810   ins_encode %{
 8811     __ block_comment("membar_release (elided)");
 8812   %}
 8813   ins_pipe(pipe_serial);
 8814 %}
 8815 
 8816 instruct membar_release() %{
 8817   match(MemBarRelease);
 8818   ins_cost(VOLATILE_REF_COST);
 8819 
 8820   format %{ "membar_release\n\t"
 8821             "dmb ish" %}
 8822 
 8823   ins_encode %{
 8824     __ block_comment("membar_release");
 8825     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8826   %}
 8827   ins_pipe(pipe_serial);
 8828 %}
 8829 
 8830 instruct membar_storestore() %{
 8831   match(MemBarStoreStore);
 8832   match(StoreStoreFence);
 8833   ins_cost(VOLATILE_REF_COST);
 8834 
 8835   format %{ "MEMBAR-store-store" %}
 8836 
 8837   ins_encode %{
 8838     __ membar(Assembler::StoreStore);
 8839   %}
 8840   ins_pipe(pipe_serial);
 8841 %}
 8842 
 8843 instruct membar_release_lock() %{
 8844   match(MemBarReleaseLock);
 8845   ins_cost(VOLATILE_REF_COST);
 8846 
 8847   format %{ "membar_release_lock (elided)" %}
 8848 
 8849   ins_encode %{
 8850     __ block_comment("membar_release_lock (elided)");
 8851   %}
 8852 
 8853   ins_pipe(pipe_serial);
 8854 %}
 8855 
 8856 instruct unnecessary_membar_volatile() %{
 8857   predicate(unnecessary_volatile(n));
 8858   match(MemBarVolatile);
 8859   ins_cost(0);
 8860 
 8861   format %{ "membar_volatile (elided)" %}
 8862 
 8863   ins_encode %{
 8864     __ block_comment("membar_volatile (elided)");
 8865   %}
 8866 
 8867   ins_pipe(pipe_serial);
 8868 %}
 8869 
 8870 instruct membar_volatile() %{
 8871   match(MemBarVolatile);
 8872   ins_cost(VOLATILE_REF_COST*100);
 8873 
 8874   format %{ "membar_volatile\n\t"
 8875              "dmb ish"%}
 8876 
 8877   ins_encode %{
 8878     __ block_comment("membar_volatile");
 8879     __ membar(Assembler::StoreLoad);
 8880   %}
 8881 
 8882   ins_pipe(pipe_serial);
 8883 %}
 8884 
 8885 // ============================================================================
 8886 // Cast/Convert Instructions
 8887 
 8888 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8889   match(Set dst (CastX2P src));
 8890 
 8891   ins_cost(INSN_COST);
 8892   format %{ "mov $dst, $src\t# long -> ptr" %}
 8893 
 8894   ins_encode %{
 8895     if ($dst$$reg != $src$$reg) {
 8896       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8897     }
 8898   %}
 8899 
 8900   ins_pipe(ialu_reg);
 8901 %}
 8902 
 8903 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8904   match(Set dst (CastP2X src));
 8905 
 8906   ins_cost(INSN_COST);
 8907   format %{ "mov $dst, $src\t# ptr -> long" %}
 8908 
 8909   ins_encode %{
 8910     if ($dst$$reg != $src$$reg) {
 8911       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8912     }
 8913   %}
 8914 
 8915   ins_pipe(ialu_reg);
 8916 %}
 8917 
 8918 // Convert oop into int for vectors alignment masking
 8919 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8920   match(Set dst (ConvL2I (CastP2X src)));
 8921 
 8922   ins_cost(INSN_COST);
 8923   format %{ "movw $dst, $src\t# ptr -> int" %}
 8924   ins_encode %{
 8925     __ movw($dst$$Register, $src$$Register);
 8926   %}
 8927 
 8928   ins_pipe(ialu_reg);
 8929 %}
 8930 
 8931 // Convert compressed oop into int for vectors alignment masking
 8932 // in case of 32bit oops (heap < 4Gb).
 8933 instruct convN2I(iRegINoSp dst, iRegN src)
 8934 %{
 8935   predicate(CompressedOops::shift() == 0);
 8936   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8937 
 8938   ins_cost(INSN_COST);
 8939   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8940   ins_encode %{
 8941     __ movw($dst$$Register, $src$$Register);
 8942   %}
 8943 
 8944   ins_pipe(ialu_reg);
 8945 %}
 8946 
 8947 
 8948 // Convert oop pointer into compressed form
 8949 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8950   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8951   match(Set dst (EncodeP src));
 8952   effect(KILL cr);
 8953   ins_cost(INSN_COST * 3);
 8954   format %{ "encode_heap_oop $dst, $src" %}
 8955   ins_encode %{
 8956     Register s = $src$$Register;
 8957     Register d = $dst$$Register;
 8958     __ encode_heap_oop(d, s);
 8959   %}
 8960   ins_pipe(ialu_reg);
 8961 %}
 8962 
 8963 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8964   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8965   match(Set dst (EncodeP src));
 8966   ins_cost(INSN_COST * 3);
 8967   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8968   ins_encode %{
 8969     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8970   %}
 8971   ins_pipe(ialu_reg);
 8972 %}
 8973 
 8974 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8975   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8976             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8977   match(Set dst (DecodeN src));
 8978   ins_cost(INSN_COST * 3);
 8979   format %{ "decode_heap_oop $dst, $src" %}
 8980   ins_encode %{
 8981     Register s = $src$$Register;
 8982     Register d = $dst$$Register;
 8983     __ decode_heap_oop(d, s);
 8984   %}
 8985   ins_pipe(ialu_reg);
 8986 %}
 8987 
 8988 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8989   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8990             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8991   match(Set dst (DecodeN src));
 8992   ins_cost(INSN_COST * 3);
 8993   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8994   ins_encode %{
 8995     Register s = $src$$Register;
 8996     Register d = $dst$$Register;
 8997     __ decode_heap_oop_not_null(d, s);
 8998   %}
 8999   ins_pipe(ialu_reg);
 9000 %}
 9001 
 9002 // n.b. AArch64 implementations of encode_klass_not_null and
 9003 // decode_klass_not_null do not modify the flags register so, unlike
 9004 // Intel, we don't kill CR as a side effect here
 9005 
 9006 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 9007   match(Set dst (EncodePKlass src));
 9008 
 9009   ins_cost(INSN_COST * 3);
 9010   format %{ "encode_klass_not_null $dst,$src" %}
 9011 
 9012   ins_encode %{
 9013     Register src_reg = as_Register($src$$reg);
 9014     Register dst_reg = as_Register($dst$$reg);
 9015     __ encode_klass_not_null(dst_reg, src_reg);
 9016   %}
 9017 
 9018    ins_pipe(ialu_reg);
 9019 %}
 9020 
 9021 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 9022   match(Set dst (DecodeNKlass src));
 9023 
 9024   ins_cost(INSN_COST * 3);
 9025   format %{ "decode_klass_not_null $dst,$src" %}
 9026 
 9027   ins_encode %{
 9028     Register src_reg = as_Register($src$$reg);
 9029     Register dst_reg = as_Register($dst$$reg);
 9030     if (dst_reg != src_reg) {
 9031       __ decode_klass_not_null(dst_reg, src_reg);
 9032     } else {
 9033       __ decode_klass_not_null(dst_reg);
 9034     }
 9035   %}
 9036 
 9037    ins_pipe(ialu_reg);
 9038 %}
 9039 
 9040 instruct checkCastPP(iRegPNoSp dst)
 9041 %{
 9042   match(Set dst (CheckCastPP dst));
 9043 
 9044   size(0);
 9045   format %{ "# checkcastPP of $dst" %}
 9046   ins_encode(/* empty encoding */);
 9047   ins_pipe(pipe_class_empty);
 9048 %}
 9049 
 9050 instruct castPP(iRegPNoSp dst)
 9051 %{
 9052   match(Set dst (CastPP dst));
 9053 
 9054   size(0);
 9055   format %{ "# castPP of $dst" %}
 9056   ins_encode(/* empty encoding */);
 9057   ins_pipe(pipe_class_empty);
 9058 %}
 9059 
 9060 instruct castII(iRegI dst)
 9061 %{
 9062   match(Set dst (CastII dst));
 9063 
 9064   size(0);
 9065   format %{ "# castII of $dst" %}
 9066   ins_encode(/* empty encoding */);
 9067   ins_cost(0);
 9068   ins_pipe(pipe_class_empty);
 9069 %}
 9070 
 9071 instruct castLL(iRegL dst)
 9072 %{
 9073   match(Set dst (CastLL dst));
 9074 
 9075   size(0);
 9076   format %{ "# castLL of $dst" %}
 9077   ins_encode(/* empty encoding */);
 9078   ins_cost(0);
 9079   ins_pipe(pipe_class_empty);
 9080 %}
 9081 
 9082 instruct castFF(vRegF dst)
 9083 %{
 9084   match(Set dst (CastFF dst));
 9085 
 9086   size(0);
 9087   format %{ "# castFF of $dst" %}
 9088   ins_encode(/* empty encoding */);
 9089   ins_cost(0);
 9090   ins_pipe(pipe_class_empty);
 9091 %}
 9092 
 9093 instruct castDD(vRegD dst)
 9094 %{
 9095   match(Set dst (CastDD dst));
 9096 
 9097   size(0);
 9098   format %{ "# castDD of $dst" %}
 9099   ins_encode(/* empty encoding */);
 9100   ins_cost(0);
 9101   ins_pipe(pipe_class_empty);
 9102 %}
 9103 
 9104 instruct castVVD(vecD dst)
 9105 %{
 9106   match(Set dst (CastVV dst));
 9107 
 9108   size(0);
 9109   format %{ "# castVV of $dst" %}
 9110   ins_encode(/* empty encoding */);
 9111   ins_cost(0);
 9112   ins_pipe(pipe_class_empty);
 9113 %}
 9114 
 9115 instruct castVVX(vecX dst)
 9116 %{
 9117   match(Set dst (CastVV dst));
 9118 
 9119   size(0);
 9120   format %{ "# castVV of $dst" %}
 9121   ins_encode(/* empty encoding */);
 9122   ins_cost(0);
 9123   ins_pipe(pipe_class_empty);
 9124 %}
 9125 
 9126 instruct castVV(vReg dst)
 9127 %{
 9128   match(Set dst (CastVV dst));
 9129 
 9130   size(0);
 9131   format %{ "# castVV of $dst" %}
 9132   ins_encode(/* empty encoding */);
 9133   ins_cost(0);
 9134   ins_pipe(pipe_class_empty);
 9135 %}
 9136 
 9137 instruct castVVMask(pRegGov dst)
 9138 %{
 9139   match(Set dst (CastVV dst));
 9140 
 9141   size(0);
 9142   format %{ "# castVV of $dst" %}
 9143   ins_encode(/* empty encoding */);
 9144   ins_cost(0);
 9145   ins_pipe(pipe_class_empty);
 9146 %}
 9147 
 9148 // ============================================================================
 9149 // Atomic operation instructions
 9150 //
 9151 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9152 // Store{PIL}Conditional instructions using a normal load for the
 9153 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9154 //
 9155 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9156 // pair to lock object allocations from Eden space when not using
 9157 // TLABs.
 9158 //
 9159 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9160 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9161 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9162 // only for 64-bit.
 9163 //
 9164 // We implement LoadPLocked and StorePLocked instructions using,
 9165 // respectively the AArch64 hw load-exclusive and store-conditional
 9166 // instructions. Whereas we must implement each of
 9167 // Store{IL}Conditional using a CAS which employs a pair of
 9168 // instructions comprising a load-exclusive followed by a
 9169 // store-conditional.
 9170 
 9171 
 9172 // Locked-load (linked load) of the current heap-top
 9173 // used when updating the eden heap top
 9174 // implemented using ldaxr on AArch64
 9175 
 9176 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9177 %{
 9178   match(Set dst (LoadPLocked mem));
 9179 
 9180   ins_cost(VOLATILE_REF_COST);
 9181 
 9182   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9183 
 9184   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9185 
 9186   ins_pipe(pipe_serial);
 9187 %}
 9188 
 9189 // Conditional-store of the updated heap-top.
 9190 // Used during allocation of the shared heap.
 9191 // Sets flag (EQ) on success.
 9192 // implemented using stlxr on AArch64.
 9193 
 9194 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9195 %{
 9196   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9197 
 9198   ins_cost(VOLATILE_REF_COST);
 9199 
 9200  // TODO
 9201  // do we need to do a store-conditional release or can we just use a
 9202  // plain store-conditional?
 9203 
 9204   format %{
 9205     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9206     "cmpw rscratch1, zr\t# EQ on successful write"
 9207   %}
 9208 
 9209   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9210 
 9211   ins_pipe(pipe_serial);
 9212 %}
 9213 
 9214 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9215 %{
 9216   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9217 
 9218   ins_cost(VOLATILE_REF_COST);
 9219 
 9220   format %{
 9221     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9222     "cmpw rscratch1, zr\t# EQ on successful write"
 9223   %}
 9224 
 9225   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9226 
 9227   ins_pipe(pipe_slow);
 9228 %}
 9229 
 9230 // storeIConditional also has acquire semantics, for no better reason
 9231 // than matching storeLConditional.  At the time of writing this
 9232 // comment storeIConditional was not used anywhere by AArch64.
 9233 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9234 %{
 9235   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9236 
 9237   ins_cost(VOLATILE_REF_COST);
 9238 
 9239   format %{
 9240     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9241     "cmpw rscratch1, zr\t# EQ on successful write"
 9242   %}
 9243 
 9244   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9245 
 9246   ins_pipe(pipe_slow);
 9247 %}
 9248 
 9249 // standard CompareAndSwapX when we are using barriers
 9250 // these have higher priority than the rules selected by a predicate
 9251 
 9252 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9253 // can't match them
 9254 
 9255 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9256 
 9257   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9258   ins_cost(2 * VOLATILE_REF_COST);
 9259 
 9260   effect(KILL cr);
 9261 
 9262   format %{
 9263     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9264     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9265   %}
 9266 
 9267   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9268             aarch64_enc_cset_eq(res));
 9269 
 9270   ins_pipe(pipe_slow);
 9271 %}
 9272 
 9273 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9274 
 9275   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9276   ins_cost(2 * VOLATILE_REF_COST);
 9277 
 9278   effect(KILL cr);
 9279 
 9280   format %{
 9281     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9282     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9283   %}
 9284 
 9285   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9286             aarch64_enc_cset_eq(res));
 9287 
 9288   ins_pipe(pipe_slow);
 9289 %}
 9290 
 9291 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9292 
 9293   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9294   ins_cost(2 * VOLATILE_REF_COST);
 9295 
 9296   effect(KILL cr);
 9297 
 9298  format %{
 9299     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9300     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9301  %}
 9302 
 9303  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9304             aarch64_enc_cset_eq(res));
 9305 
 9306   ins_pipe(pipe_slow);
 9307 %}
 9308 
 9309 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9310 
 9311   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9312   ins_cost(2 * VOLATILE_REF_COST);
 9313 
 9314   effect(KILL cr);
 9315 
 9316  format %{
 9317     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9318     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9319  %}
 9320 
 9321  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9322             aarch64_enc_cset_eq(res));
 9323 
 9324   ins_pipe(pipe_slow);
 9325 %}
 9326 
 9327 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9328 
 9329   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9330   predicate(n->as_LoadStore()->barrier_data() == 0);
 9331   ins_cost(2 * VOLATILE_REF_COST);
 9332 
 9333   effect(KILL cr);
 9334 
 9335  format %{
 9336     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9337     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9338  %}
 9339 
 9340  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9341             aarch64_enc_cset_eq(res));
 9342 
 9343   ins_pipe(pipe_slow);
 9344 %}
 9345 
 9346 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9347 
 9348   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9349   ins_cost(2 * VOLATILE_REF_COST);
 9350 
 9351   effect(KILL cr);
 9352 
 9353  format %{
 9354     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9355     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9356  %}
 9357 
 9358  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9359             aarch64_enc_cset_eq(res));
 9360 
 9361   ins_pipe(pipe_slow);
 9362 %}
 9363 
 9364 // alternative CompareAndSwapX when we are eliding barriers
 9365 
 9366 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9367 
 9368   predicate(needs_acquiring_load_exclusive(n));
 9369   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9370   ins_cost(VOLATILE_REF_COST);
 9371 
 9372   effect(KILL cr);
 9373 
 9374   format %{
 9375     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9376     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9377   %}
 9378 
 9379   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9380             aarch64_enc_cset_eq(res));
 9381 
 9382   ins_pipe(pipe_slow);
 9383 %}
 9384 
 9385 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9386 
 9387   predicate(needs_acquiring_load_exclusive(n));
 9388   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9389   ins_cost(VOLATILE_REF_COST);
 9390 
 9391   effect(KILL cr);
 9392 
 9393   format %{
 9394     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9395     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9396   %}
 9397 
 9398   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9399             aarch64_enc_cset_eq(res));
 9400 
 9401   ins_pipe(pipe_slow);
 9402 %}
 9403 
 9404 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9405 
 9406   predicate(needs_acquiring_load_exclusive(n));
 9407   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9408   ins_cost(VOLATILE_REF_COST);
 9409 
 9410   effect(KILL cr);
 9411 
 9412  format %{
 9413     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9414     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9415  %}
 9416 
 9417  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9418             aarch64_enc_cset_eq(res));
 9419 
 9420   ins_pipe(pipe_slow);
 9421 %}
 9422 
 9423 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9424 
 9425   predicate(needs_acquiring_load_exclusive(n));
 9426   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9427   ins_cost(VOLATILE_REF_COST);
 9428 
 9429   effect(KILL cr);
 9430 
 9431  format %{
 9432     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9433     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9434  %}
 9435 
 9436  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9437             aarch64_enc_cset_eq(res));
 9438 
 9439   ins_pipe(pipe_slow);
 9440 %}
 9441 
 9442 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9443 
 9444   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9445   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9446   ins_cost(VOLATILE_REF_COST);
 9447 
 9448   effect(KILL cr);
 9449 
 9450  format %{
 9451     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9452     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9453  %}
 9454 
 9455  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9456             aarch64_enc_cset_eq(res));
 9457 
 9458   ins_pipe(pipe_slow);
 9459 %}
 9460 
 9461 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9462 
 9463   predicate(needs_acquiring_load_exclusive(n));
 9464   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9465   ins_cost(VOLATILE_REF_COST);
 9466 
 9467   effect(KILL cr);
 9468 
 9469  format %{
 9470     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9471     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9472  %}
 9473 
 9474  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9475             aarch64_enc_cset_eq(res));
 9476 
 9477   ins_pipe(pipe_slow);
 9478 %}
 9479 
 9480 
 9481 // ---------------------------------------------------------------------
 9482 
 9483 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9484 
 9485 // Sundry CAS operations.  Note that release is always true,
 9486 // regardless of the memory ordering of the CAS.  This is because we
 9487 // need the volatile case to be sequentially consistent but there is
 9488 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9489 // can't check the type of memory ordering here, so we always emit a
 9490 // STLXR.
 9491 
 9492 // This section is generated from aarch64_ad_cas.m4
 9493 
 9494 
 9495 
 9496 // This pattern is generated automatically from cas.m4.
 9497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9498 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9499 
 9500   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9501   ins_cost(2 * VOLATILE_REF_COST);
 9502   effect(TEMP_DEF res, KILL cr);
 9503   format %{
 9504     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9505   %}
 9506   ins_encode %{
 9507     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9508                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9509                /*weak*/ false, $res$$Register);
 9510     __ sxtbw($res$$Register, $res$$Register);
 9511   %}
 9512   ins_pipe(pipe_slow);
 9513 %}
 9514 
 9515 // This pattern is generated automatically from cas.m4.
 9516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9517 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9518 
 9519   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9520   ins_cost(2 * VOLATILE_REF_COST);
 9521   effect(TEMP_DEF res, KILL cr);
 9522   format %{
 9523     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9524   %}
 9525   ins_encode %{
 9526     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9527                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9528                /*weak*/ false, $res$$Register);
 9529     __ sxthw($res$$Register, $res$$Register);
 9530   %}
 9531   ins_pipe(pipe_slow);
 9532 %}
 9533 
 9534 // This pattern is generated automatically from cas.m4.
 9535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9536 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9537 
 9538   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9539   ins_cost(2 * VOLATILE_REF_COST);
 9540   effect(TEMP_DEF res, KILL cr);
 9541   format %{
 9542     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9543   %}
 9544   ins_encode %{
 9545     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9546                Assembler::word, /*acquire*/ false, /*release*/ true,
 9547                /*weak*/ false, $res$$Register);
 9548   %}
 9549   ins_pipe(pipe_slow);
 9550 %}
 9551 
 9552 // This pattern is generated automatically from cas.m4.
 9553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9554 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9555 
 9556   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9557   ins_cost(2 * VOLATILE_REF_COST);
 9558   effect(TEMP_DEF res, KILL cr);
 9559   format %{
 9560     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9561   %}
 9562   ins_encode %{
 9563     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9564                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9565                /*weak*/ false, $res$$Register);
 9566   %}
 9567   ins_pipe(pipe_slow);
 9568 %}
 9569 
 9570 // This pattern is generated automatically from cas.m4.
 9571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9572 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9573 
 9574   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9575   ins_cost(2 * VOLATILE_REF_COST);
 9576   effect(TEMP_DEF res, KILL cr);
 9577   format %{
 9578     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9579   %}
 9580   ins_encode %{
 9581     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9582                Assembler::word, /*acquire*/ false, /*release*/ true,
 9583                /*weak*/ false, $res$$Register);
 9584   %}
 9585   ins_pipe(pipe_slow);
 9586 %}
 9587 
 9588 // This pattern is generated automatically from cas.m4.
 9589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9590 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9591   predicate(n->as_LoadStore()->barrier_data() == 0);
 9592   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9593   ins_cost(2 * VOLATILE_REF_COST);
 9594   effect(TEMP_DEF res, KILL cr);
 9595   format %{
 9596     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9597   %}
 9598   ins_encode %{
 9599     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9600                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9601                /*weak*/ false, $res$$Register);
 9602   %}
 9603   ins_pipe(pipe_slow);
 9604 %}
 9605 
 9606 // This pattern is generated automatically from cas.m4.
 9607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9608 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9609   predicate(needs_acquiring_load_exclusive(n));
 9610   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9611   ins_cost(VOLATILE_REF_COST);
 9612   effect(TEMP_DEF res, KILL cr);
 9613   format %{
 9614     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9615   %}
 9616   ins_encode %{
 9617     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9618                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9619                /*weak*/ false, $res$$Register);
 9620     __ sxtbw($res$$Register, $res$$Register);
 9621   %}
 9622   ins_pipe(pipe_slow);
 9623 %}
 9624 
 9625 // This pattern is generated automatically from cas.m4.
 9626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9627 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9628   predicate(needs_acquiring_load_exclusive(n));
 9629   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9630   ins_cost(VOLATILE_REF_COST);
 9631   effect(TEMP_DEF res, KILL cr);
 9632   format %{
 9633     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9634   %}
 9635   ins_encode %{
 9636     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9637                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9638                /*weak*/ false, $res$$Register);
 9639     __ sxthw($res$$Register, $res$$Register);
 9640   %}
 9641   ins_pipe(pipe_slow);
 9642 %}
 9643 
 9644 // This pattern is generated automatically from cas.m4.
 9645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9646 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9647   predicate(needs_acquiring_load_exclusive(n));
 9648   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9649   ins_cost(VOLATILE_REF_COST);
 9650   effect(TEMP_DEF res, KILL cr);
 9651   format %{
 9652     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9653   %}
 9654   ins_encode %{
 9655     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9656                Assembler::word, /*acquire*/ true, /*release*/ true,
 9657                /*weak*/ false, $res$$Register);
 9658   %}
 9659   ins_pipe(pipe_slow);
 9660 %}
 9661 
 9662 // This pattern is generated automatically from cas.m4.
 9663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9664 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9665   predicate(needs_acquiring_load_exclusive(n));
 9666   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9667   ins_cost(VOLATILE_REF_COST);
 9668   effect(TEMP_DEF res, KILL cr);
 9669   format %{
 9670     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9671   %}
 9672   ins_encode %{
 9673     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9674                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9675                /*weak*/ false, $res$$Register);
 9676   %}
 9677   ins_pipe(pipe_slow);
 9678 %}
 9679 
 9680 // This pattern is generated automatically from cas.m4.
 9681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9682 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9683   predicate(needs_acquiring_load_exclusive(n));
 9684   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9685   ins_cost(VOLATILE_REF_COST);
 9686   effect(TEMP_DEF res, KILL cr);
 9687   format %{
 9688     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9689   %}
 9690   ins_encode %{
 9691     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9692                Assembler::word, /*acquire*/ true, /*release*/ true,
 9693                /*weak*/ false, $res$$Register);
 9694   %}
 9695   ins_pipe(pipe_slow);
 9696 %}
 9697 
 9698 // This pattern is generated automatically from cas.m4.
 9699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9700 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9701   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9702   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9703   ins_cost(VOLATILE_REF_COST);
 9704   effect(TEMP_DEF res, KILL cr);
 9705   format %{
 9706     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9707   %}
 9708   ins_encode %{
 9709     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9710                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9711                /*weak*/ false, $res$$Register);
 9712   %}
 9713   ins_pipe(pipe_slow);
 9714 %}
 9715 
 9716 // This pattern is generated automatically from cas.m4.
 9717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9718 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9719 
 9720   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9721   ins_cost(2 * VOLATILE_REF_COST);
 9722   effect(KILL cr);
 9723   format %{
 9724     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9725     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9726   %}
 9727   ins_encode %{
 9728     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9729                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9730                /*weak*/ true, noreg);
 9731     __ csetw($res$$Register, Assembler::EQ);
 9732   %}
 9733   ins_pipe(pipe_slow);
 9734 %}
 9735 
 9736 // This pattern is generated automatically from cas.m4.
 9737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9738 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9739 
 9740   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9741   ins_cost(2 * VOLATILE_REF_COST);
 9742   effect(KILL cr);
 9743   format %{
 9744     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9745     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9746   %}
 9747   ins_encode %{
 9748     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9749                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9750                /*weak*/ true, noreg);
 9751     __ csetw($res$$Register, Assembler::EQ);
 9752   %}
 9753   ins_pipe(pipe_slow);
 9754 %}
 9755 
 9756 // This pattern is generated automatically from cas.m4.
 9757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9758 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9759 
 9760   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9761   ins_cost(2 * VOLATILE_REF_COST);
 9762   effect(KILL cr);
 9763   format %{
 9764     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9765     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9766   %}
 9767   ins_encode %{
 9768     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9769                Assembler::word, /*acquire*/ false, /*release*/ true,
 9770                /*weak*/ true, noreg);
 9771     __ csetw($res$$Register, Assembler::EQ);
 9772   %}
 9773   ins_pipe(pipe_slow);
 9774 %}
 9775 
 9776 // This pattern is generated automatically from cas.m4.
 9777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9778 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9779 
 9780   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9781   ins_cost(2 * VOLATILE_REF_COST);
 9782   effect(KILL cr);
 9783   format %{
 9784     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9785     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9786   %}
 9787   ins_encode %{
 9788     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9789                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9790                /*weak*/ true, noreg);
 9791     __ csetw($res$$Register, Assembler::EQ);
 9792   %}
 9793   ins_pipe(pipe_slow);
 9794 %}
 9795 
 9796 // This pattern is generated automatically from cas.m4.
 9797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9798 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9799 
 9800   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9801   ins_cost(2 * VOLATILE_REF_COST);
 9802   effect(KILL cr);
 9803   format %{
 9804     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9805     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9806   %}
 9807   ins_encode %{
 9808     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9809                Assembler::word, /*acquire*/ false, /*release*/ true,
 9810                /*weak*/ true, noreg);
 9811     __ csetw($res$$Register, Assembler::EQ);
 9812   %}
 9813   ins_pipe(pipe_slow);
 9814 %}
 9815 
 9816 // This pattern is generated automatically from cas.m4.
 9817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9818 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9819   predicate(n->as_LoadStore()->barrier_data() == 0);
 9820   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9821   ins_cost(2 * VOLATILE_REF_COST);
 9822   effect(KILL cr);
 9823   format %{
 9824     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9825     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9826   %}
 9827   ins_encode %{
 9828     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9829                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9830                /*weak*/ true, noreg);
 9831     __ csetw($res$$Register, Assembler::EQ);
 9832   %}
 9833   ins_pipe(pipe_slow);
 9834 %}
 9835 
 9836 // This pattern is generated automatically from cas.m4.
 9837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9838 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9839   predicate(needs_acquiring_load_exclusive(n));
 9840   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9841   ins_cost(VOLATILE_REF_COST);
 9842   effect(KILL cr);
 9843   format %{
 9844     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9845     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9846   %}
 9847   ins_encode %{
 9848     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9849                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9850                /*weak*/ true, noreg);
 9851     __ csetw($res$$Register, Assembler::EQ);
 9852   %}
 9853   ins_pipe(pipe_slow);
 9854 %}
 9855 
 9856 // This pattern is generated automatically from cas.m4.
 9857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9858 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9859   predicate(needs_acquiring_load_exclusive(n));
 9860   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9861   ins_cost(VOLATILE_REF_COST);
 9862   effect(KILL cr);
 9863   format %{
 9864     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9865     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9866   %}
 9867   ins_encode %{
 9868     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9869                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9870                /*weak*/ true, noreg);
 9871     __ csetw($res$$Register, Assembler::EQ);
 9872   %}
 9873   ins_pipe(pipe_slow);
 9874 %}
 9875 
 9876 // This pattern is generated automatically from cas.m4.
 9877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9878 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9879   predicate(needs_acquiring_load_exclusive(n));
 9880   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9881   ins_cost(VOLATILE_REF_COST);
 9882   effect(KILL cr);
 9883   format %{
 9884     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9885     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9886   %}
 9887   ins_encode %{
 9888     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9889                Assembler::word, /*acquire*/ true, /*release*/ true,
 9890                /*weak*/ true, noreg);
 9891     __ csetw($res$$Register, Assembler::EQ);
 9892   %}
 9893   ins_pipe(pipe_slow);
 9894 %}
 9895 
 9896 // This pattern is generated automatically from cas.m4.
 9897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9898 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9899   predicate(needs_acquiring_load_exclusive(n));
 9900   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9901   ins_cost(VOLATILE_REF_COST);
 9902   effect(KILL cr);
 9903   format %{
 9904     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9905     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9906   %}
 9907   ins_encode %{
 9908     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9909                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9910                /*weak*/ true, noreg);
 9911     __ csetw($res$$Register, Assembler::EQ);
 9912   %}
 9913   ins_pipe(pipe_slow);
 9914 %}
 9915 
 9916 // This pattern is generated automatically from cas.m4.
 9917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9918 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9919   predicate(needs_acquiring_load_exclusive(n));
 9920   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9921   ins_cost(VOLATILE_REF_COST);
 9922   effect(KILL cr);
 9923   format %{
 9924     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9925     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9926   %}
 9927   ins_encode %{
 9928     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9929                Assembler::word, /*acquire*/ true, /*release*/ true,
 9930                /*weak*/ true, noreg);
 9931     __ csetw($res$$Register, Assembler::EQ);
 9932   %}
 9933   ins_pipe(pipe_slow);
 9934 %}
 9935 
 9936 // This pattern is generated automatically from cas.m4.
 9937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9938 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9939   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9940   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9941   ins_cost(VOLATILE_REF_COST);
 9942   effect(KILL cr);
 9943   format %{
 9944     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9945     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9946   %}
 9947   ins_encode %{
 9948     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9949                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9950                /*weak*/ true, noreg);
 9951     __ csetw($res$$Register, Assembler::EQ);
 9952   %}
 9953   ins_pipe(pipe_slow);
 9954 %}
 9955 
 9956 // END This section of the file is automatically generated. Do not edit --------------
 9957 // ---------------------------------------------------------------------
 9958 
 9959 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9960   match(Set prev (GetAndSetI mem newv));
 9961   ins_cost(2 * VOLATILE_REF_COST);
 9962   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9963   ins_encode %{
 9964     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9965   %}
 9966   ins_pipe(pipe_serial);
 9967 %}
 9968 
 9969 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9970   match(Set prev (GetAndSetL mem newv));
 9971   ins_cost(2 * VOLATILE_REF_COST);
 9972   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9973   ins_encode %{
 9974     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9975   %}
 9976   ins_pipe(pipe_serial);
 9977 %}
 9978 
 9979 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9980   match(Set prev (GetAndSetN mem newv));
 9981   ins_cost(2 * VOLATILE_REF_COST);
 9982   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9983   ins_encode %{
 9984     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9985   %}
 9986   ins_pipe(pipe_serial);
 9987 %}
 9988 
 9989 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9990   predicate(n->as_LoadStore()->barrier_data() == 0);
 9991   match(Set prev (GetAndSetP mem newv));
 9992   ins_cost(2 * VOLATILE_REF_COST);
 9993   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9994   ins_encode %{
 9995     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9996   %}
 9997   ins_pipe(pipe_serial);
 9998 %}
 9999 
10000 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
10001   predicate(needs_acquiring_load_exclusive(n));
10002   match(Set prev (GetAndSetI mem newv));
10003   ins_cost(VOLATILE_REF_COST);
10004   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
10005   ins_encode %{
10006     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10007   %}
10008   ins_pipe(pipe_serial);
10009 %}
10010 
10011 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
10012   predicate(needs_acquiring_load_exclusive(n));
10013   match(Set prev (GetAndSetL mem newv));
10014   ins_cost(VOLATILE_REF_COST);
10015   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
10016   ins_encode %{
10017     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10018   %}
10019   ins_pipe(pipe_serial);
10020 %}
10021 
10022 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
10023   predicate(needs_acquiring_load_exclusive(n));
10024   match(Set prev (GetAndSetN mem newv));
10025   ins_cost(VOLATILE_REF_COST);
10026   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
10027   ins_encode %{
10028     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10029   %}
10030   ins_pipe(pipe_serial);
10031 %}
10032 
10033 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
10034   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
10035   match(Set prev (GetAndSetP mem newv));
10036   ins_cost(VOLATILE_REF_COST);
10037   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
10038   ins_encode %{
10039     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10040   %}
10041   ins_pipe(pipe_serial);
10042 %}
10043 
10044 
10045 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
10046   match(Set newval (GetAndAddL mem incr));
10047   ins_cost(2 * VOLATILE_REF_COST + 1);
10048   format %{ "get_and_addL $newval, [$mem], $incr" %}
10049   ins_encode %{
10050     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
10051   %}
10052   ins_pipe(pipe_serial);
10053 %}
10054 
10055 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
10056   predicate(n->as_LoadStore()->result_not_used());
10057   match(Set dummy (GetAndAddL mem incr));
10058   ins_cost(2 * VOLATILE_REF_COST);
10059   format %{ "get_and_addL [$mem], $incr" %}
10060   ins_encode %{
10061     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
10062   %}
10063   ins_pipe(pipe_serial);
10064 %}
10065 
10066 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10067   match(Set newval (GetAndAddL mem incr));
10068   ins_cost(2 * VOLATILE_REF_COST + 1);
10069   format %{ "get_and_addL $newval, [$mem], $incr" %}
10070   ins_encode %{
10071     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
10072   %}
10073   ins_pipe(pipe_serial);
10074 %}
10075 
10076 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
10077   predicate(n->as_LoadStore()->result_not_used());
10078   match(Set dummy (GetAndAddL mem incr));
10079   ins_cost(2 * VOLATILE_REF_COST);
10080   format %{ "get_and_addL [$mem], $incr" %}
10081   ins_encode %{
10082     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
10083   %}
10084   ins_pipe(pipe_serial);
10085 %}
10086 
10087 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10088   match(Set newval (GetAndAddI mem incr));
10089   ins_cost(2 * VOLATILE_REF_COST + 1);
10090   format %{ "get_and_addI $newval, [$mem], $incr" %}
10091   ins_encode %{
10092     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10093   %}
10094   ins_pipe(pipe_serial);
10095 %}
10096 
10097 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10098   predicate(n->as_LoadStore()->result_not_used());
10099   match(Set dummy (GetAndAddI mem incr));
10100   ins_cost(2 * VOLATILE_REF_COST);
10101   format %{ "get_and_addI [$mem], $incr" %}
10102   ins_encode %{
10103     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10104   %}
10105   ins_pipe(pipe_serial);
10106 %}
10107 
10108 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10109   match(Set newval (GetAndAddI mem incr));
10110   ins_cost(2 * VOLATILE_REF_COST + 1);
10111   format %{ "get_and_addI $newval, [$mem], $incr" %}
10112   ins_encode %{
10113     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10114   %}
10115   ins_pipe(pipe_serial);
10116 %}
10117 
10118 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10119   predicate(n->as_LoadStore()->result_not_used());
10120   match(Set dummy (GetAndAddI mem incr));
10121   ins_cost(2 * VOLATILE_REF_COST);
10122   format %{ "get_and_addI [$mem], $incr" %}
10123   ins_encode %{
10124     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10125   %}
10126   ins_pipe(pipe_serial);
10127 %}
10128 
10129 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10130   predicate(needs_acquiring_load_exclusive(n));
10131   match(Set newval (GetAndAddL mem incr));
10132   ins_cost(VOLATILE_REF_COST + 1);
10133   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10134   ins_encode %{
10135     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10136   %}
10137   ins_pipe(pipe_serial);
10138 %}
10139 
10140 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10141   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10142   match(Set dummy (GetAndAddL mem incr));
10143   ins_cost(VOLATILE_REF_COST);
10144   format %{ "get_and_addL_acq [$mem], $incr" %}
10145   ins_encode %{
10146     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10147   %}
10148   ins_pipe(pipe_serial);
10149 %}
10150 
10151 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10152   predicate(needs_acquiring_load_exclusive(n));
10153   match(Set newval (GetAndAddL mem incr));
10154   ins_cost(VOLATILE_REF_COST + 1);
10155   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10156   ins_encode %{
10157     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10158   %}
10159   ins_pipe(pipe_serial);
10160 %}
10161 
10162 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10163   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10164   match(Set dummy (GetAndAddL mem incr));
10165   ins_cost(VOLATILE_REF_COST);
10166   format %{ "get_and_addL_acq [$mem], $incr" %}
10167   ins_encode %{
10168     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10169   %}
10170   ins_pipe(pipe_serial);
10171 %}
10172 
10173 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10174   predicate(needs_acquiring_load_exclusive(n));
10175   match(Set newval (GetAndAddI mem incr));
10176   ins_cost(VOLATILE_REF_COST + 1);
10177   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10178   ins_encode %{
10179     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10180   %}
10181   ins_pipe(pipe_serial);
10182 %}
10183 
10184 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10185   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10186   match(Set dummy (GetAndAddI mem incr));
10187   ins_cost(VOLATILE_REF_COST);
10188   format %{ "get_and_addI_acq [$mem], $incr" %}
10189   ins_encode %{
10190     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10191   %}
10192   ins_pipe(pipe_serial);
10193 %}
10194 
10195 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10196   predicate(needs_acquiring_load_exclusive(n));
10197   match(Set newval (GetAndAddI mem incr));
10198   ins_cost(VOLATILE_REF_COST + 1);
10199   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10200   ins_encode %{
10201     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10202   %}
10203   ins_pipe(pipe_serial);
10204 %}
10205 
10206 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10207   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10208   match(Set dummy (GetAndAddI mem incr));
10209   ins_cost(VOLATILE_REF_COST);
10210   format %{ "get_and_addI_acq [$mem], $incr" %}
10211   ins_encode %{
10212     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10213   %}
10214   ins_pipe(pipe_serial);
10215 %}
10216 
10217 // Manifest a CmpL result in an integer register.
10218 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10219 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10220 %{
10221   match(Set dst (CmpL3 src1 src2));
10222   effect(KILL flags);
10223 
10224   ins_cost(INSN_COST * 6);
10225   format %{
10226       "cmp $src1, $src2"
10227       "csetw $dst, ne"
10228       "cnegw $dst, lt"
10229   %}
10230   // format %{ "CmpL3 $dst, $src1, $src2" %}
10231   ins_encode %{
10232     __ cmp($src1$$Register, $src2$$Register);
10233     __ csetw($dst$$Register, Assembler::NE);
10234     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10235   %}
10236 
10237   ins_pipe(pipe_class_default);
10238 %}
10239 
10240 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10241 %{
10242   match(Set dst (CmpL3 src1 src2));
10243   effect(KILL flags);
10244 
10245   ins_cost(INSN_COST * 6);
10246   format %{
10247       "cmp $src1, $src2"
10248       "csetw $dst, ne"
10249       "cnegw $dst, lt"
10250   %}
10251   ins_encode %{
10252     int32_t con = (int32_t)$src2$$constant;
10253      if (con < 0) {
10254       __ adds(zr, $src1$$Register, -con);
10255     } else {
10256       __ subs(zr, $src1$$Register, con);
10257     }
10258     __ csetw($dst$$Register, Assembler::NE);
10259     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10260   %}
10261 
10262   ins_pipe(pipe_class_default);
10263 %}
10264 
10265 // ============================================================================
10266 // Conditional Move Instructions
10267 
10268 // n.b. we have identical rules for both a signed compare op (cmpOp)
10269 // and an unsigned compare op (cmpOpU). it would be nice if we could
10270 // define an op class which merged both inputs and use it to type the
10271 // argument to a single rule. unfortunatelyt his fails because the
10272 // opclass does not live up to the COND_INTER interface of its
10273 // component operands. When the generic code tries to negate the
10274 // operand it ends up running the generci Machoper::negate method
10275 // which throws a ShouldNotHappen. So, we have to provide two flavours
10276 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10277 
10278 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10279   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10280 
10281   ins_cost(INSN_COST * 2);
10282   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10283 
10284   ins_encode %{
10285     __ cselw(as_Register($dst$$reg),
10286              as_Register($src2$$reg),
10287              as_Register($src1$$reg),
10288              (Assembler::Condition)$cmp$$cmpcode);
10289   %}
10290 
10291   ins_pipe(icond_reg_reg);
10292 %}
10293 
10294 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10295   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10296 
10297   ins_cost(INSN_COST * 2);
10298   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10299 
10300   ins_encode %{
10301     __ cselw(as_Register($dst$$reg),
10302              as_Register($src2$$reg),
10303              as_Register($src1$$reg),
10304              (Assembler::Condition)$cmp$$cmpcode);
10305   %}
10306 
10307   ins_pipe(icond_reg_reg);
10308 %}
10309 
10310 // special cases where one arg is zero
10311 
10312 // n.b. this is selected in preference to the rule above because it
10313 // avoids loading constant 0 into a source register
10314 
10315 // TODO
10316 // we ought only to be able to cull one of these variants as the ideal
10317 // transforms ought always to order the zero consistently (to left/right?)
10318 
10319 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10320   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10321 
10322   ins_cost(INSN_COST * 2);
10323   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10324 
10325   ins_encode %{
10326     __ cselw(as_Register($dst$$reg),
10327              as_Register($src$$reg),
10328              zr,
10329              (Assembler::Condition)$cmp$$cmpcode);
10330   %}
10331 
10332   ins_pipe(icond_reg);
10333 %}
10334 
10335 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10336   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10337 
10338   ins_cost(INSN_COST * 2);
10339   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10340 
10341   ins_encode %{
10342     __ cselw(as_Register($dst$$reg),
10343              as_Register($src$$reg),
10344              zr,
10345              (Assembler::Condition)$cmp$$cmpcode);
10346   %}
10347 
10348   ins_pipe(icond_reg);
10349 %}
10350 
10351 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10352   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10353 
10354   ins_cost(INSN_COST * 2);
10355   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10356 
10357   ins_encode %{
10358     __ cselw(as_Register($dst$$reg),
10359              zr,
10360              as_Register($src$$reg),
10361              (Assembler::Condition)$cmp$$cmpcode);
10362   %}
10363 
10364   ins_pipe(icond_reg);
10365 %}
10366 
10367 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10368   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10369 
10370   ins_cost(INSN_COST * 2);
10371   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10372 
10373   ins_encode %{
10374     __ cselw(as_Register($dst$$reg),
10375              zr,
10376              as_Register($src$$reg),
10377              (Assembler::Condition)$cmp$$cmpcode);
10378   %}
10379 
10380   ins_pipe(icond_reg);
10381 %}
10382 
10383 // special case for creating a boolean 0 or 1
10384 
10385 // n.b. this is selected in preference to the rule above because it
10386 // avoids loading constants 0 and 1 into a source register
10387 
10388 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10389   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10390 
10391   ins_cost(INSN_COST * 2);
10392   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10393 
10394   ins_encode %{
10395     // equivalently
10396     // cset(as_Register($dst$$reg),
10397     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10398     __ csincw(as_Register($dst$$reg),
10399              zr,
10400              zr,
10401              (Assembler::Condition)$cmp$$cmpcode);
10402   %}
10403 
10404   ins_pipe(icond_none);
10405 %}
10406 
10407 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10408   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10409 
10410   ins_cost(INSN_COST * 2);
10411   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10412 
10413   ins_encode %{
10414     // equivalently
10415     // cset(as_Register($dst$$reg),
10416     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10417     __ csincw(as_Register($dst$$reg),
10418              zr,
10419              zr,
10420              (Assembler::Condition)$cmp$$cmpcode);
10421   %}
10422 
10423   ins_pipe(icond_none);
10424 %}
10425 
10426 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10427   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10428 
10429   ins_cost(INSN_COST * 2);
10430   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10431 
10432   ins_encode %{
10433     __ csel(as_Register($dst$$reg),
10434             as_Register($src2$$reg),
10435             as_Register($src1$$reg),
10436             (Assembler::Condition)$cmp$$cmpcode);
10437   %}
10438 
10439   ins_pipe(icond_reg_reg);
10440 %}
10441 
10442 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10443   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10444 
10445   ins_cost(INSN_COST * 2);
10446   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10447 
10448   ins_encode %{
10449     __ csel(as_Register($dst$$reg),
10450             as_Register($src2$$reg),
10451             as_Register($src1$$reg),
10452             (Assembler::Condition)$cmp$$cmpcode);
10453   %}
10454 
10455   ins_pipe(icond_reg_reg);
10456 %}
10457 
10458 // special cases where one arg is zero
10459 
10460 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10461   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10462 
10463   ins_cost(INSN_COST * 2);
10464   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10465 
10466   ins_encode %{
10467     __ csel(as_Register($dst$$reg),
10468             zr,
10469             as_Register($src$$reg),
10470             (Assembler::Condition)$cmp$$cmpcode);
10471   %}
10472 
10473   ins_pipe(icond_reg);
10474 %}
10475 
10476 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10477   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10478 
10479   ins_cost(INSN_COST * 2);
10480   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10481 
10482   ins_encode %{
10483     __ csel(as_Register($dst$$reg),
10484             zr,
10485             as_Register($src$$reg),
10486             (Assembler::Condition)$cmp$$cmpcode);
10487   %}
10488 
10489   ins_pipe(icond_reg);
10490 %}
10491 
10492 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10493   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10494 
10495   ins_cost(INSN_COST * 2);
10496   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10497 
10498   ins_encode %{
10499     __ csel(as_Register($dst$$reg),
10500             as_Register($src$$reg),
10501             zr,
10502             (Assembler::Condition)$cmp$$cmpcode);
10503   %}
10504 
10505   ins_pipe(icond_reg);
10506 %}
10507 
10508 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10509   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10510 
10511   ins_cost(INSN_COST * 2);
10512   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10513 
10514   ins_encode %{
10515     __ csel(as_Register($dst$$reg),
10516             as_Register($src$$reg),
10517             zr,
10518             (Assembler::Condition)$cmp$$cmpcode);
10519   %}
10520 
10521   ins_pipe(icond_reg);
10522 %}
10523 
10524 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10525   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10526 
10527   ins_cost(INSN_COST * 2);
10528   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10529 
10530   ins_encode %{
10531     __ csel(as_Register($dst$$reg),
10532             as_Register($src2$$reg),
10533             as_Register($src1$$reg),
10534             (Assembler::Condition)$cmp$$cmpcode);
10535   %}
10536 
10537   ins_pipe(icond_reg_reg);
10538 %}
10539 
10540 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10541   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10542 
10543   ins_cost(INSN_COST * 2);
10544   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10545 
10546   ins_encode %{
10547     __ csel(as_Register($dst$$reg),
10548             as_Register($src2$$reg),
10549             as_Register($src1$$reg),
10550             (Assembler::Condition)$cmp$$cmpcode);
10551   %}
10552 
10553   ins_pipe(icond_reg_reg);
10554 %}
10555 
10556 // special cases where one arg is zero
10557 
10558 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10559   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10560 
10561   ins_cost(INSN_COST * 2);
10562   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10563 
10564   ins_encode %{
10565     __ csel(as_Register($dst$$reg),
10566             zr,
10567             as_Register($src$$reg),
10568             (Assembler::Condition)$cmp$$cmpcode);
10569   %}
10570 
10571   ins_pipe(icond_reg);
10572 %}
10573 
10574 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10575   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10576 
10577   ins_cost(INSN_COST * 2);
10578   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10579 
10580   ins_encode %{
10581     __ csel(as_Register($dst$$reg),
10582             zr,
10583             as_Register($src$$reg),
10584             (Assembler::Condition)$cmp$$cmpcode);
10585   %}
10586 
10587   ins_pipe(icond_reg);
10588 %}
10589 
10590 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10591   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10592 
10593   ins_cost(INSN_COST * 2);
10594   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10595 
10596   ins_encode %{
10597     __ csel(as_Register($dst$$reg),
10598             as_Register($src$$reg),
10599             zr,
10600             (Assembler::Condition)$cmp$$cmpcode);
10601   %}
10602 
10603   ins_pipe(icond_reg);
10604 %}
10605 
10606 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10607   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10608 
10609   ins_cost(INSN_COST * 2);
10610   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10611 
10612   ins_encode %{
10613     __ csel(as_Register($dst$$reg),
10614             as_Register($src$$reg),
10615             zr,
10616             (Assembler::Condition)$cmp$$cmpcode);
10617   %}
10618 
10619   ins_pipe(icond_reg);
10620 %}
10621 
10622 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10623   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10624 
10625   ins_cost(INSN_COST * 2);
10626   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10627 
10628   ins_encode %{
10629     __ cselw(as_Register($dst$$reg),
10630              as_Register($src2$$reg),
10631              as_Register($src1$$reg),
10632              (Assembler::Condition)$cmp$$cmpcode);
10633   %}
10634 
10635   ins_pipe(icond_reg_reg);
10636 %}
10637 
10638 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10639   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10640 
10641   ins_cost(INSN_COST * 2);
10642   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10643 
10644   ins_encode %{
10645     __ cselw(as_Register($dst$$reg),
10646              as_Register($src2$$reg),
10647              as_Register($src1$$reg),
10648              (Assembler::Condition)$cmp$$cmpcode);
10649   %}
10650 
10651   ins_pipe(icond_reg_reg);
10652 %}
10653 
10654 // special cases where one arg is zero
10655 
10656 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10657   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10658 
10659   ins_cost(INSN_COST * 2);
10660   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10661 
10662   ins_encode %{
10663     __ cselw(as_Register($dst$$reg),
10664              zr,
10665              as_Register($src$$reg),
10666              (Assembler::Condition)$cmp$$cmpcode);
10667   %}
10668 
10669   ins_pipe(icond_reg);
10670 %}
10671 
10672 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10673   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10674 
10675   ins_cost(INSN_COST * 2);
10676   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10677 
10678   ins_encode %{
10679     __ cselw(as_Register($dst$$reg),
10680              zr,
10681              as_Register($src$$reg),
10682              (Assembler::Condition)$cmp$$cmpcode);
10683   %}
10684 
10685   ins_pipe(icond_reg);
10686 %}
10687 
10688 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10689   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10690 
10691   ins_cost(INSN_COST * 2);
10692   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10693 
10694   ins_encode %{
10695     __ cselw(as_Register($dst$$reg),
10696              as_Register($src$$reg),
10697              zr,
10698              (Assembler::Condition)$cmp$$cmpcode);
10699   %}
10700 
10701   ins_pipe(icond_reg);
10702 %}
10703 
10704 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10705   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10706 
10707   ins_cost(INSN_COST * 2);
10708   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10709 
10710   ins_encode %{
10711     __ cselw(as_Register($dst$$reg),
10712              as_Register($src$$reg),
10713              zr,
10714              (Assembler::Condition)$cmp$$cmpcode);
10715   %}
10716 
10717   ins_pipe(icond_reg);
10718 %}
10719 
10720 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10721 %{
10722   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10723 
10724   ins_cost(INSN_COST * 3);
10725 
10726   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10727   ins_encode %{
10728     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10729     __ fcsels(as_FloatRegister($dst$$reg),
10730               as_FloatRegister($src2$$reg),
10731               as_FloatRegister($src1$$reg),
10732               cond);
10733   %}
10734 
10735   ins_pipe(fp_cond_reg_reg_s);
10736 %}
10737 
10738 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10739 %{
10740   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10741 
10742   ins_cost(INSN_COST * 3);
10743 
10744   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10745   ins_encode %{
10746     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10747     __ fcsels(as_FloatRegister($dst$$reg),
10748               as_FloatRegister($src2$$reg),
10749               as_FloatRegister($src1$$reg),
10750               cond);
10751   %}
10752 
10753   ins_pipe(fp_cond_reg_reg_s);
10754 %}
10755 
10756 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10757 %{
10758   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10759 
10760   ins_cost(INSN_COST * 3);
10761 
10762   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10763   ins_encode %{
10764     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10765     __ fcseld(as_FloatRegister($dst$$reg),
10766               as_FloatRegister($src2$$reg),
10767               as_FloatRegister($src1$$reg),
10768               cond);
10769   %}
10770 
10771   ins_pipe(fp_cond_reg_reg_d);
10772 %}
10773 
10774 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10775 %{
10776   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10777 
10778   ins_cost(INSN_COST * 3);
10779 
10780   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10781   ins_encode %{
10782     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10783     __ fcseld(as_FloatRegister($dst$$reg),
10784               as_FloatRegister($src2$$reg),
10785               as_FloatRegister($src1$$reg),
10786               cond);
10787   %}
10788 
10789   ins_pipe(fp_cond_reg_reg_d);
10790 %}
10791 
10792 // ============================================================================
10793 // Arithmetic Instructions
10794 //
10795 
10796 // Integer Addition
10797 
10798 // TODO
10799 // these currently employ operations which do not set CR and hence are
10800 // not flagged as killing CR but we would like to isolate the cases
10801 // where we want to set flags from those where we don't. need to work
10802 // out how to do that.
10803 
10804 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10805   match(Set dst (AddI src1 src2));
10806 
10807   ins_cost(INSN_COST);
10808   format %{ "addw  $dst, $src1, $src2" %}
10809 
10810   ins_encode %{
10811     __ addw(as_Register($dst$$reg),
10812             as_Register($src1$$reg),
10813             as_Register($src2$$reg));
10814   %}
10815 
10816   ins_pipe(ialu_reg_reg);
10817 %}
10818 
10819 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10820   match(Set dst (AddI src1 src2));
10821 
10822   ins_cost(INSN_COST);
10823   format %{ "addw $dst, $src1, $src2" %}
10824 
10825   // use opcode to indicate that this is an add not a sub
10826   opcode(0x0);
10827 
10828   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10829 
10830   ins_pipe(ialu_reg_imm);
10831 %}
10832 
10833 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10834   match(Set dst (AddI (ConvL2I src1) src2));
10835 
10836   ins_cost(INSN_COST);
10837   format %{ "addw $dst, $src1, $src2" %}
10838 
10839   // use opcode to indicate that this is an add not a sub
10840   opcode(0x0);
10841 
10842   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10843 
10844   ins_pipe(ialu_reg_imm);
10845 %}
10846 
10847 // Pointer Addition
10848 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10849   match(Set dst (AddP src1 src2));
10850 
10851   ins_cost(INSN_COST);
10852   format %{ "add $dst, $src1, $src2\t# ptr" %}
10853 
10854   ins_encode %{
10855     __ add(as_Register($dst$$reg),
10856            as_Register($src1$$reg),
10857            as_Register($src2$$reg));
10858   %}
10859 
10860   ins_pipe(ialu_reg_reg);
10861 %}
10862 
10863 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10864   match(Set dst (AddP src1 (ConvI2L src2)));
10865 
10866   ins_cost(1.9 * INSN_COST);
10867   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10868 
10869   ins_encode %{
10870     __ add(as_Register($dst$$reg),
10871            as_Register($src1$$reg),
10872            as_Register($src2$$reg), ext::sxtw);
10873   %}
10874 
10875   ins_pipe(ialu_reg_reg);
10876 %}
10877 
10878 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10879   match(Set dst (AddP src1 (LShiftL src2 scale)));
10880 
10881   ins_cost(1.9 * INSN_COST);
10882   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10883 
10884   ins_encode %{
10885     __ lea(as_Register($dst$$reg),
10886            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10887                    Address::lsl($scale$$constant)));
10888   %}
10889 
10890   ins_pipe(ialu_reg_reg_shift);
10891 %}
10892 
10893 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10894   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10895 
10896   ins_cost(1.9 * INSN_COST);
10897   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10898 
10899   ins_encode %{
10900     __ lea(as_Register($dst$$reg),
10901            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10902                    Address::sxtw($scale$$constant)));
10903   %}
10904 
10905   ins_pipe(ialu_reg_reg_shift);
10906 %}
10907 
10908 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10909   match(Set dst (LShiftL (ConvI2L src) scale));
10910 
10911   ins_cost(INSN_COST);
10912   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10913 
10914   ins_encode %{
10915     __ sbfiz(as_Register($dst$$reg),
10916           as_Register($src$$reg),
10917           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10918   %}
10919 
10920   ins_pipe(ialu_reg_shift);
10921 %}
10922 
10923 // Pointer Immediate Addition
10924 // n.b. this needs to be more expensive than using an indirect memory
10925 // operand
10926 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10927   match(Set dst (AddP src1 src2));
10928 
10929   ins_cost(INSN_COST);
10930   format %{ "add $dst, $src1, $src2\t# ptr" %}
10931 
10932   // use opcode to indicate that this is an add not a sub
10933   opcode(0x0);
10934 
10935   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10936 
10937   ins_pipe(ialu_reg_imm);
10938 %}
10939 
10940 // Long Addition
10941 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10942 
10943   match(Set dst (AddL src1 src2));
10944 
10945   ins_cost(INSN_COST);
10946   format %{ "add  $dst, $src1, $src2" %}
10947 
10948   ins_encode %{
10949     __ add(as_Register($dst$$reg),
10950            as_Register($src1$$reg),
10951            as_Register($src2$$reg));
10952   %}
10953 
10954   ins_pipe(ialu_reg_reg);
10955 %}
10956 
10957 // No constant pool entries requiredLong Immediate Addition.
10958 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10959   match(Set dst (AddL src1 src2));
10960 
10961   ins_cost(INSN_COST);
10962   format %{ "add $dst, $src1, $src2" %}
10963 
10964   // use opcode to indicate that this is an add not a sub
10965   opcode(0x0);
10966 
10967   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10968 
10969   ins_pipe(ialu_reg_imm);
10970 %}
10971 
10972 // Integer Subtraction
10973 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10974   match(Set dst (SubI src1 src2));
10975 
10976   ins_cost(INSN_COST);
10977   format %{ "subw  $dst, $src1, $src2" %}
10978 
10979   ins_encode %{
10980     __ subw(as_Register($dst$$reg),
10981             as_Register($src1$$reg),
10982             as_Register($src2$$reg));
10983   %}
10984 
10985   ins_pipe(ialu_reg_reg);
10986 %}
10987 
10988 // Immediate Subtraction
10989 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10990   match(Set dst (SubI src1 src2));
10991 
10992   ins_cost(INSN_COST);
10993   format %{ "subw $dst, $src1, $src2" %}
10994 
10995   // use opcode to indicate that this is a sub not an add
10996   opcode(0x1);
10997 
10998   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10999 
11000   ins_pipe(ialu_reg_imm);
11001 %}
11002 
11003 // Long Subtraction
11004 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11005 
11006   match(Set dst (SubL src1 src2));
11007 
11008   ins_cost(INSN_COST);
11009   format %{ "sub  $dst, $src1, $src2" %}
11010 
11011   ins_encode %{
11012     __ sub(as_Register($dst$$reg),
11013            as_Register($src1$$reg),
11014            as_Register($src2$$reg));
11015   %}
11016 
11017   ins_pipe(ialu_reg_reg);
11018 %}
11019 
11020 // No constant pool entries requiredLong Immediate Subtraction.
11021 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
11022   match(Set dst (SubL src1 src2));
11023 
11024   ins_cost(INSN_COST);
11025   format %{ "sub$dst, $src1, $src2" %}
11026 
11027   // use opcode to indicate that this is a sub not an add
11028   opcode(0x1);
11029 
11030   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
11031 
11032   ins_pipe(ialu_reg_imm);
11033 %}
11034 
11035 // Integer Negation (special case for sub)
11036 
11037 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
11038   match(Set dst (SubI zero src));
11039 
11040   ins_cost(INSN_COST);
11041   format %{ "negw $dst, $src\t# int" %}
11042 
11043   ins_encode %{
11044     __ negw(as_Register($dst$$reg),
11045             as_Register($src$$reg));
11046   %}
11047 
11048   ins_pipe(ialu_reg);
11049 %}
11050 
11051 // Long Negation
11052 
11053 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
11054   match(Set dst (SubL zero src));
11055 
11056   ins_cost(INSN_COST);
11057   format %{ "neg $dst, $src\t# long" %}
11058 
11059   ins_encode %{
11060     __ neg(as_Register($dst$$reg),
11061            as_Register($src$$reg));
11062   %}
11063 
11064   ins_pipe(ialu_reg);
11065 %}
11066 
11067 // Integer Multiply
11068 
11069 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11070   match(Set dst (MulI src1 src2));
11071 
11072   ins_cost(INSN_COST * 3);
11073   format %{ "mulw  $dst, $src1, $src2" %}
11074 
11075   ins_encode %{
11076     __ mulw(as_Register($dst$$reg),
11077             as_Register($src1$$reg),
11078             as_Register($src2$$reg));
11079   %}
11080 
11081   ins_pipe(imul_reg_reg);
11082 %}
11083 
11084 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11085   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
11086 
11087   ins_cost(INSN_COST * 3);
11088   format %{ "smull  $dst, $src1, $src2" %}
11089 
11090   ins_encode %{
11091     __ smull(as_Register($dst$$reg),
11092              as_Register($src1$$reg),
11093              as_Register($src2$$reg));
11094   %}
11095 
11096   ins_pipe(imul_reg_reg);
11097 %}
11098 
11099 // Long Multiply
11100 
11101 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11102   match(Set dst (MulL src1 src2));
11103 
11104   ins_cost(INSN_COST * 5);
11105   format %{ "mul  $dst, $src1, $src2" %}
11106 
11107   ins_encode %{
11108     __ mul(as_Register($dst$$reg),
11109            as_Register($src1$$reg),
11110            as_Register($src2$$reg));
11111   %}
11112 
11113   ins_pipe(lmul_reg_reg);
11114 %}
11115 
11116 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11117 %{
11118   match(Set dst (MulHiL src1 src2));
11119 
11120   ins_cost(INSN_COST * 7);
11121   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
11122 
11123   ins_encode %{
11124     __ smulh(as_Register($dst$$reg),
11125              as_Register($src1$$reg),
11126              as_Register($src2$$reg));
11127   %}
11128 
11129   ins_pipe(lmul_reg_reg);
11130 %}
11131 
11132 // Combined Integer Multiply & Add/Sub
11133 
11134 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11135   match(Set dst (AddI src3 (MulI src1 src2)));
11136 
11137   ins_cost(INSN_COST * 3);
11138   format %{ "madd  $dst, $src1, $src2, $src3" %}
11139 
11140   ins_encode %{
11141     __ maddw(as_Register($dst$$reg),
11142              as_Register($src1$$reg),
11143              as_Register($src2$$reg),
11144              as_Register($src3$$reg));
11145   %}
11146 
11147   ins_pipe(imac_reg_reg);
11148 %}
11149 
11150 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11151   match(Set dst (SubI src3 (MulI src1 src2)));
11152 
11153   ins_cost(INSN_COST * 3);
11154   format %{ "msub  $dst, $src1, $src2, $src3" %}
11155 
11156   ins_encode %{
11157     __ msubw(as_Register($dst$$reg),
11158              as_Register($src1$$reg),
11159              as_Register($src2$$reg),
11160              as_Register($src3$$reg));
11161   %}
11162 
11163   ins_pipe(imac_reg_reg);
11164 %}
11165 
11166 // Combined Integer Multiply & Neg
11167 
11168 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11169   match(Set dst (MulI (SubI zero src1) src2));
11170 
11171   ins_cost(INSN_COST * 3);
11172   format %{ "mneg  $dst, $src1, $src2" %}
11173 
11174   ins_encode %{
11175     __ mnegw(as_Register($dst$$reg),
11176              as_Register($src1$$reg),
11177              as_Register($src2$$reg));
11178   %}
11179 
11180   ins_pipe(imac_reg_reg);
11181 %}
11182 
11183 // Combined Long Multiply & Add/Sub
11184 
11185 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11186   match(Set dst (AddL src3 (MulL src1 src2)));
11187 
11188   ins_cost(INSN_COST * 5);
11189   format %{ "madd  $dst, $src1, $src2, $src3" %}
11190 
11191   ins_encode %{
11192     __ madd(as_Register($dst$$reg),
11193             as_Register($src1$$reg),
11194             as_Register($src2$$reg),
11195             as_Register($src3$$reg));
11196   %}
11197 
11198   ins_pipe(lmac_reg_reg);
11199 %}
11200 
11201 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11202   match(Set dst (SubL src3 (MulL src1 src2)));
11203 
11204   ins_cost(INSN_COST * 5);
11205   format %{ "msub  $dst, $src1, $src2, $src3" %}
11206 
11207   ins_encode %{
11208     __ msub(as_Register($dst$$reg),
11209             as_Register($src1$$reg),
11210             as_Register($src2$$reg),
11211             as_Register($src3$$reg));
11212   %}
11213 
11214   ins_pipe(lmac_reg_reg);
11215 %}
11216 
11217 // Combined Long Multiply & Neg
11218 
11219 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11220   match(Set dst (MulL (SubL zero src1) src2));
11221 
11222   ins_cost(INSN_COST * 5);
11223   format %{ "mneg  $dst, $src1, $src2" %}
11224 
11225   ins_encode %{
11226     __ mneg(as_Register($dst$$reg),
11227             as_Register($src1$$reg),
11228             as_Register($src2$$reg));
11229   %}
11230 
11231   ins_pipe(lmac_reg_reg);
11232 %}
11233 
11234 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11235 
11236 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11237   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11238 
11239   ins_cost(INSN_COST * 3);
11240   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11241 
11242   ins_encode %{
11243     __ smaddl(as_Register($dst$$reg),
11244               as_Register($src1$$reg),
11245               as_Register($src2$$reg),
11246               as_Register($src3$$reg));
11247   %}
11248 
11249   ins_pipe(imac_reg_reg);
11250 %}
11251 
11252 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11253   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11254 
11255   ins_cost(INSN_COST * 3);
11256   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11257 
11258   ins_encode %{
11259     __ smsubl(as_Register($dst$$reg),
11260               as_Register($src1$$reg),
11261               as_Register($src2$$reg),
11262               as_Register($src3$$reg));
11263   %}
11264 
11265   ins_pipe(imac_reg_reg);
11266 %}
11267 
11268 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11269   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11270 
11271   ins_cost(INSN_COST * 3);
11272   format %{ "smnegl  $dst, $src1, $src2" %}
11273 
11274   ins_encode %{
11275     __ smnegl(as_Register($dst$$reg),
11276               as_Register($src1$$reg),
11277               as_Register($src2$$reg));
11278   %}
11279 
11280   ins_pipe(imac_reg_reg);
11281 %}
11282 
11283 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11284 
11285 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11286   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11287 
11288   ins_cost(INSN_COST * 5);
11289   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11290             "maddw $dst, $src3, $src4, rscratch1" %}
11291 
11292   ins_encode %{
11293     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11294     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11295 
11296   ins_pipe(imac_reg_reg);
11297 %}
11298 
11299 // Integer Divide
11300 
11301 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11302   match(Set dst (DivI src1 src2));
11303 
11304   ins_cost(INSN_COST * 19);
11305   format %{ "sdivw  $dst, $src1, $src2" %}
11306 
11307   ins_encode(aarch64_enc_divw(dst, src1, src2));
11308   ins_pipe(idiv_reg_reg);
11309 %}
11310 
11311 // Long Divide
11312 
11313 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11314   match(Set dst (DivL src1 src2));
11315 
11316   ins_cost(INSN_COST * 35);
11317   format %{ "sdiv   $dst, $src1, $src2" %}
11318 
11319   ins_encode(aarch64_enc_div(dst, src1, src2));
11320   ins_pipe(ldiv_reg_reg);
11321 %}
11322 
11323 // Integer Remainder
11324 
11325 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11326   match(Set dst (ModI src1 src2));
11327 
11328   ins_cost(INSN_COST * 22);
11329   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11330             "msubw($dst, rscratch1, $src2, $src1" %}
11331 
11332   ins_encode(aarch64_enc_modw(dst, src1, src2));
11333   ins_pipe(idiv_reg_reg);
11334 %}
11335 
11336 // Long Remainder
11337 
11338 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11339   match(Set dst (ModL src1 src2));
11340 
11341   ins_cost(INSN_COST * 38);
11342   format %{ "sdiv   rscratch1, $src1, $src2\n"
11343             "msub($dst, rscratch1, $src2, $src1" %}
11344 
11345   ins_encode(aarch64_enc_mod(dst, src1, src2));
11346   ins_pipe(ldiv_reg_reg);
11347 %}
11348 
11349 // Integer Shifts
11350 
11351 // Shift Left Register
11352 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11353   match(Set dst (LShiftI src1 src2));
11354 
11355   ins_cost(INSN_COST * 2);
11356   format %{ "lslvw  $dst, $src1, $src2" %}
11357 
11358   ins_encode %{
11359     __ lslvw(as_Register($dst$$reg),
11360              as_Register($src1$$reg),
11361              as_Register($src2$$reg));
11362   %}
11363 
11364   ins_pipe(ialu_reg_reg_vshift);
11365 %}
11366 
11367 // Shift Left Immediate
11368 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11369   match(Set dst (LShiftI src1 src2));
11370 
11371   ins_cost(INSN_COST);
11372   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11373 
11374   ins_encode %{
11375     __ lslw(as_Register($dst$$reg),
11376             as_Register($src1$$reg),
11377             $src2$$constant & 0x1f);
11378   %}
11379 
11380   ins_pipe(ialu_reg_shift);
11381 %}
11382 
11383 // Shift Right Logical Register
11384 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11385   match(Set dst (URShiftI src1 src2));
11386 
11387   ins_cost(INSN_COST * 2);
11388   format %{ "lsrvw  $dst, $src1, $src2" %}
11389 
11390   ins_encode %{
11391     __ lsrvw(as_Register($dst$$reg),
11392              as_Register($src1$$reg),
11393              as_Register($src2$$reg));
11394   %}
11395 
11396   ins_pipe(ialu_reg_reg_vshift);
11397 %}
11398 
11399 // Shift Right Logical Immediate
11400 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11401   match(Set dst (URShiftI src1 src2));
11402 
11403   ins_cost(INSN_COST);
11404   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11405 
11406   ins_encode %{
11407     __ lsrw(as_Register($dst$$reg),
11408             as_Register($src1$$reg),
11409             $src2$$constant & 0x1f);
11410   %}
11411 
11412   ins_pipe(ialu_reg_shift);
11413 %}
11414 
11415 // Shift Right Arithmetic Register
11416 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11417   match(Set dst (RShiftI src1 src2));
11418 
11419   ins_cost(INSN_COST * 2);
11420   format %{ "asrvw  $dst, $src1, $src2" %}
11421 
11422   ins_encode %{
11423     __ asrvw(as_Register($dst$$reg),
11424              as_Register($src1$$reg),
11425              as_Register($src2$$reg));
11426   %}
11427 
11428   ins_pipe(ialu_reg_reg_vshift);
11429 %}
11430 
11431 // Shift Right Arithmetic Immediate
11432 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11433   match(Set dst (RShiftI src1 src2));
11434 
11435   ins_cost(INSN_COST);
11436   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11437 
11438   ins_encode %{
11439     __ asrw(as_Register($dst$$reg),
11440             as_Register($src1$$reg),
11441             $src2$$constant & 0x1f);
11442   %}
11443 
11444   ins_pipe(ialu_reg_shift);
11445 %}
11446 
11447 // Combined Int Mask and Right Shift (using UBFM)
11448 // TODO
11449 
11450 // Long Shifts
11451 
11452 // Shift Left Register
11453 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11454   match(Set dst (LShiftL src1 src2));
11455 
11456   ins_cost(INSN_COST * 2);
11457   format %{ "lslv  $dst, $src1, $src2" %}
11458 
11459   ins_encode %{
11460     __ lslv(as_Register($dst$$reg),
11461             as_Register($src1$$reg),
11462             as_Register($src2$$reg));
11463   %}
11464 
11465   ins_pipe(ialu_reg_reg_vshift);
11466 %}
11467 
11468 // Shift Left Immediate
11469 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11470   match(Set dst (LShiftL src1 src2));
11471 
11472   ins_cost(INSN_COST);
11473   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11474 
11475   ins_encode %{
11476     __ lsl(as_Register($dst$$reg),
11477             as_Register($src1$$reg),
11478             $src2$$constant & 0x3f);
11479   %}
11480 
11481   ins_pipe(ialu_reg_shift);
11482 %}
11483 
11484 // Shift Right Logical Register
11485 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11486   match(Set dst (URShiftL src1 src2));
11487 
11488   ins_cost(INSN_COST * 2);
11489   format %{ "lsrv  $dst, $src1, $src2" %}
11490 
11491   ins_encode %{
11492     __ lsrv(as_Register($dst$$reg),
11493             as_Register($src1$$reg),
11494             as_Register($src2$$reg));
11495   %}
11496 
11497   ins_pipe(ialu_reg_reg_vshift);
11498 %}
11499 
11500 // Shift Right Logical Immediate
11501 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11502   match(Set dst (URShiftL src1 src2));
11503 
11504   ins_cost(INSN_COST);
11505   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11506 
11507   ins_encode %{
11508     __ lsr(as_Register($dst$$reg),
11509            as_Register($src1$$reg),
11510            $src2$$constant & 0x3f);
11511   %}
11512 
11513   ins_pipe(ialu_reg_shift);
11514 %}
11515 
11516 // A special-case pattern for card table stores.
11517 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11518   match(Set dst (URShiftL (CastP2X src1) src2));
11519 
11520   ins_cost(INSN_COST);
11521   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11522 
11523   ins_encode %{
11524     __ lsr(as_Register($dst$$reg),
11525            as_Register($src1$$reg),
11526            $src2$$constant & 0x3f);
11527   %}
11528 
11529   ins_pipe(ialu_reg_shift);
11530 %}
11531 
11532 // Shift Right Arithmetic Register
11533 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11534   match(Set dst (RShiftL src1 src2));
11535 
11536   ins_cost(INSN_COST * 2);
11537   format %{ "asrv  $dst, $src1, $src2" %}
11538 
11539   ins_encode %{
11540     __ asrv(as_Register($dst$$reg),
11541             as_Register($src1$$reg),
11542             as_Register($src2$$reg));
11543   %}
11544 
11545   ins_pipe(ialu_reg_reg_vshift);
11546 %}
11547 
11548 // Shift Right Arithmetic Immediate
11549 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11550   match(Set dst (RShiftL src1 src2));
11551 
11552   ins_cost(INSN_COST);
11553   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11554 
11555   ins_encode %{
11556     __ asr(as_Register($dst$$reg),
11557            as_Register($src1$$reg),
11558            $src2$$constant & 0x3f);
11559   %}
11560 
11561   ins_pipe(ialu_reg_shift);
11562 %}
11563 
11564 // BEGIN This section of the file is automatically generated. Do not edit --------------
11565 // This section is generated from aarch64_ad.m4
11566 
11567 
11568 // This pattern is automatically generated from aarch64_ad.m4
11569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11570 instruct regL_not_reg(iRegLNoSp dst,
11571                          iRegL src1, immL_M1 m1,
11572                          rFlagsReg cr) %{
11573   match(Set dst (XorL src1 m1));
11574   ins_cost(INSN_COST);
11575   format %{ "eon  $dst, $src1, zr" %}
11576 
11577   ins_encode %{
11578     __ eon(as_Register($dst$$reg),
11579               as_Register($src1$$reg),
11580               zr,
11581               Assembler::LSL, 0);
11582   %}
11583 
11584   ins_pipe(ialu_reg);
11585 %}
11586 
11587 // This pattern is automatically generated from aarch64_ad.m4
11588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11589 instruct regI_not_reg(iRegINoSp dst,
11590                          iRegIorL2I src1, immI_M1 m1,
11591                          rFlagsReg cr) %{
11592   match(Set dst (XorI src1 m1));
11593   ins_cost(INSN_COST);
11594   format %{ "eonw  $dst, $src1, zr" %}
11595 
11596   ins_encode %{
11597     __ eonw(as_Register($dst$$reg),
11598               as_Register($src1$$reg),
11599               zr,
11600               Assembler::LSL, 0);
11601   %}
11602 
11603   ins_pipe(ialu_reg);
11604 %}
11605 
11606 // This pattern is automatically generated from aarch64_ad.m4
11607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11608 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11609                               immI0 zero, iRegIorL2I src1, immI src2) %{
11610   match(Set dst (SubI zero (URShiftI src1 src2)));
11611 
11612   ins_cost(1.9 * INSN_COST);
11613   format %{ "negw  $dst, $src1, LSR $src2" %}
11614 
11615   ins_encode %{
11616     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11617             Assembler::LSR, $src2$$constant & 0x1f);
11618   %}
11619 
11620   ins_pipe(ialu_reg_shift);
11621 %}
11622 
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11626                               immI0 zero, iRegIorL2I src1, immI src2) %{
11627   match(Set dst (SubI zero (RShiftI src1 src2)));
11628 
11629   ins_cost(1.9 * INSN_COST);
11630   format %{ "negw  $dst, $src1, ASR $src2" %}
11631 
11632   ins_encode %{
11633     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11634             Assembler::ASR, $src2$$constant & 0x1f);
11635   %}
11636 
11637   ins_pipe(ialu_reg_shift);
11638 %}
11639 
11640 // This pattern is automatically generated from aarch64_ad.m4
11641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11642 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11643                               immI0 zero, iRegIorL2I src1, immI src2) %{
11644   match(Set dst (SubI zero (LShiftI src1 src2)));
11645 
11646   ins_cost(1.9 * INSN_COST);
11647   format %{ "negw  $dst, $src1, LSL $src2" %}
11648 
11649   ins_encode %{
11650     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11651             Assembler::LSL, $src2$$constant & 0x1f);
11652   %}
11653 
11654   ins_pipe(ialu_reg_shift);
11655 %}
11656 
11657 // This pattern is automatically generated from aarch64_ad.m4
11658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11659 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11660                               immL0 zero, iRegL src1, immI src2) %{
11661   match(Set dst (SubL zero (URShiftL src1 src2)));
11662 
11663   ins_cost(1.9 * INSN_COST);
11664   format %{ "neg  $dst, $src1, LSR $src2" %}
11665 
11666   ins_encode %{
11667     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11668             Assembler::LSR, $src2$$constant & 0x3f);
11669   %}
11670 
11671   ins_pipe(ialu_reg_shift);
11672 %}
11673 
11674 // This pattern is automatically generated from aarch64_ad.m4
11675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11676 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11677                               immL0 zero, iRegL src1, immI src2) %{
11678   match(Set dst (SubL zero (RShiftL src1 src2)));
11679 
11680   ins_cost(1.9 * INSN_COST);
11681   format %{ "neg  $dst, $src1, ASR $src2" %}
11682 
11683   ins_encode %{
11684     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11685             Assembler::ASR, $src2$$constant & 0x3f);
11686   %}
11687 
11688   ins_pipe(ialu_reg_shift);
11689 %}
11690 
11691 // This pattern is automatically generated from aarch64_ad.m4
11692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11693 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11694                               immL0 zero, iRegL src1, immI src2) %{
11695   match(Set dst (SubL zero (LShiftL src1 src2)));
11696 
11697   ins_cost(1.9 * INSN_COST);
11698   format %{ "neg  $dst, $src1, LSL $src2" %}
11699 
11700   ins_encode %{
11701     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11702             Assembler::LSL, $src2$$constant & 0x3f);
11703   %}
11704 
11705   ins_pipe(ialu_reg_shift);
11706 %}
11707 
11708 // This pattern is automatically generated from aarch64_ad.m4
11709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11710 instruct AndI_reg_not_reg(iRegINoSp dst,
11711                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11712   match(Set dst (AndI src1 (XorI src2 m1)));
11713   ins_cost(INSN_COST);
11714   format %{ "bicw  $dst, $src1, $src2" %}
11715 
11716   ins_encode %{
11717     __ bicw(as_Register($dst$$reg),
11718               as_Register($src1$$reg),
11719               as_Register($src2$$reg),
11720               Assembler::LSL, 0);
11721   %}
11722 
11723   ins_pipe(ialu_reg_reg);
11724 %}
11725 
11726 // This pattern is automatically generated from aarch64_ad.m4
11727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11728 instruct AndL_reg_not_reg(iRegLNoSp dst,
11729                          iRegL src1, iRegL src2, immL_M1 m1) %{
11730   match(Set dst (AndL src1 (XorL src2 m1)));
11731   ins_cost(INSN_COST);
11732   format %{ "bic  $dst, $src1, $src2" %}
11733 
11734   ins_encode %{
11735     __ bic(as_Register($dst$$reg),
11736               as_Register($src1$$reg),
11737               as_Register($src2$$reg),
11738               Assembler::LSL, 0);
11739   %}
11740 
11741   ins_pipe(ialu_reg_reg);
11742 %}
11743 
11744 // This pattern is automatically generated from aarch64_ad.m4
11745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11746 instruct OrI_reg_not_reg(iRegINoSp dst,
11747                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11748   match(Set dst (OrI src1 (XorI src2 m1)));
11749   ins_cost(INSN_COST);
11750   format %{ "ornw  $dst, $src1, $src2" %}
11751 
11752   ins_encode %{
11753     __ ornw(as_Register($dst$$reg),
11754               as_Register($src1$$reg),
11755               as_Register($src2$$reg),
11756               Assembler::LSL, 0);
11757   %}
11758 
11759   ins_pipe(ialu_reg_reg);
11760 %}
11761 
11762 // This pattern is automatically generated from aarch64_ad.m4
11763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11764 instruct OrL_reg_not_reg(iRegLNoSp dst,
11765                          iRegL src1, iRegL src2, immL_M1 m1) %{
11766   match(Set dst (OrL src1 (XorL src2 m1)));
11767   ins_cost(INSN_COST);
11768   format %{ "orn  $dst, $src1, $src2" %}
11769 
11770   ins_encode %{
11771     __ orn(as_Register($dst$$reg),
11772               as_Register($src1$$reg),
11773               as_Register($src2$$reg),
11774               Assembler::LSL, 0);
11775   %}
11776 
11777   ins_pipe(ialu_reg_reg);
11778 %}
11779 
11780 // This pattern is automatically generated from aarch64_ad.m4
11781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11782 instruct XorI_reg_not_reg(iRegINoSp dst,
11783                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11784   match(Set dst (XorI m1 (XorI src2 src1)));
11785   ins_cost(INSN_COST);
11786   format %{ "eonw  $dst, $src1, $src2" %}
11787 
11788   ins_encode %{
11789     __ eonw(as_Register($dst$$reg),
11790               as_Register($src1$$reg),
11791               as_Register($src2$$reg),
11792               Assembler::LSL, 0);
11793   %}
11794 
11795   ins_pipe(ialu_reg_reg);
11796 %}
11797 
11798 // This pattern is automatically generated from aarch64_ad.m4
11799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11800 instruct XorL_reg_not_reg(iRegLNoSp dst,
11801                          iRegL src1, iRegL src2, immL_M1 m1) %{
11802   match(Set dst (XorL m1 (XorL src2 src1)));
11803   ins_cost(INSN_COST);
11804   format %{ "eon  $dst, $src1, $src2" %}
11805 
11806   ins_encode %{
11807     __ eon(as_Register($dst$$reg),
11808               as_Register($src1$$reg),
11809               as_Register($src2$$reg),
11810               Assembler::LSL, 0);
11811   %}
11812 
11813   ins_pipe(ialu_reg_reg);
11814 %}
11815 
11816 // This pattern is automatically generated from aarch64_ad.m4
11817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11818 // val & (-1 ^ (val >>> shift)) ==> bicw
11819 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11820                          iRegIorL2I src1, iRegIorL2I src2,
11821                          immI src3, immI_M1 src4) %{
11822   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11823   ins_cost(1.9 * INSN_COST);
11824   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11825 
11826   ins_encode %{
11827     __ bicw(as_Register($dst$$reg),
11828               as_Register($src1$$reg),
11829               as_Register($src2$$reg),
11830               Assembler::LSR,
11831               $src3$$constant & 0x1f);
11832   %}
11833 
11834   ins_pipe(ialu_reg_reg_shift);
11835 %}
11836 
11837 // This pattern is automatically generated from aarch64_ad.m4
11838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11839 // val & (-1 ^ (val >>> shift)) ==> bic
11840 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11841                          iRegL src1, iRegL src2,
11842                          immI src3, immL_M1 src4) %{
11843   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11844   ins_cost(1.9 * INSN_COST);
11845   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11846 
11847   ins_encode %{
11848     __ bic(as_Register($dst$$reg),
11849               as_Register($src1$$reg),
11850               as_Register($src2$$reg),
11851               Assembler::LSR,
11852               $src3$$constant & 0x3f);
11853   %}
11854 
11855   ins_pipe(ialu_reg_reg_shift);
11856 %}
11857 
11858 // This pattern is automatically generated from aarch64_ad.m4
11859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11860 // val & (-1 ^ (val >> shift)) ==> bicw
11861 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11862                          iRegIorL2I src1, iRegIorL2I src2,
11863                          immI src3, immI_M1 src4) %{
11864   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11865   ins_cost(1.9 * INSN_COST);
11866   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11867 
11868   ins_encode %{
11869     __ bicw(as_Register($dst$$reg),
11870               as_Register($src1$$reg),
11871               as_Register($src2$$reg),
11872               Assembler::ASR,
11873               $src3$$constant & 0x1f);
11874   %}
11875 
11876   ins_pipe(ialu_reg_reg_shift);
11877 %}
11878 
11879 // This pattern is automatically generated from aarch64_ad.m4
11880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11881 // val & (-1 ^ (val >> shift)) ==> bic
11882 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11883                          iRegL src1, iRegL src2,
11884                          immI src3, immL_M1 src4) %{
11885   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11886   ins_cost(1.9 * INSN_COST);
11887   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11888 
11889   ins_encode %{
11890     __ bic(as_Register($dst$$reg),
11891               as_Register($src1$$reg),
11892               as_Register($src2$$reg),
11893               Assembler::ASR,
11894               $src3$$constant & 0x3f);
11895   %}
11896 
11897   ins_pipe(ialu_reg_reg_shift);
11898 %}
11899 
11900 // This pattern is automatically generated from aarch64_ad.m4
11901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11902 // val & (-1 ^ (val ror shift)) ==> bicw
11903 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11904                          iRegIorL2I src1, iRegIorL2I src2,
11905                          immI src3, immI_M1 src4) %{
11906   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11907   ins_cost(1.9 * INSN_COST);
11908   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11909 
11910   ins_encode %{
11911     __ bicw(as_Register($dst$$reg),
11912               as_Register($src1$$reg),
11913               as_Register($src2$$reg),
11914               Assembler::ROR,
11915               $src3$$constant & 0x1f);
11916   %}
11917 
11918   ins_pipe(ialu_reg_reg_shift);
11919 %}
11920 
11921 // This pattern is automatically generated from aarch64_ad.m4
11922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11923 // val & (-1 ^ (val ror shift)) ==> bic
11924 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11925                          iRegL src1, iRegL src2,
11926                          immI src3, immL_M1 src4) %{
11927   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11928   ins_cost(1.9 * INSN_COST);
11929   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11930 
11931   ins_encode %{
11932     __ bic(as_Register($dst$$reg),
11933               as_Register($src1$$reg),
11934               as_Register($src2$$reg),
11935               Assembler::ROR,
11936               $src3$$constant & 0x3f);
11937   %}
11938 
11939   ins_pipe(ialu_reg_reg_shift);
11940 %}
11941 
11942 // This pattern is automatically generated from aarch64_ad.m4
11943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11944 // val & (-1 ^ (val << shift)) ==> bicw
11945 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11946                          iRegIorL2I src1, iRegIorL2I src2,
11947                          immI src3, immI_M1 src4) %{
11948   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11949   ins_cost(1.9 * INSN_COST);
11950   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11951 
11952   ins_encode %{
11953     __ bicw(as_Register($dst$$reg),
11954               as_Register($src1$$reg),
11955               as_Register($src2$$reg),
11956               Assembler::LSL,
11957               $src3$$constant & 0x1f);
11958   %}
11959 
11960   ins_pipe(ialu_reg_reg_shift);
11961 %}
11962 
11963 // This pattern is automatically generated from aarch64_ad.m4
11964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11965 // val & (-1 ^ (val << shift)) ==> bic
11966 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11967                          iRegL src1, iRegL src2,
11968                          immI src3, immL_M1 src4) %{
11969   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11970   ins_cost(1.9 * INSN_COST);
11971   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11972 
11973   ins_encode %{
11974     __ bic(as_Register($dst$$reg),
11975               as_Register($src1$$reg),
11976               as_Register($src2$$reg),
11977               Assembler::LSL,
11978               $src3$$constant & 0x3f);
11979   %}
11980 
11981   ins_pipe(ialu_reg_reg_shift);
11982 %}
11983 
11984 // This pattern is automatically generated from aarch64_ad.m4
11985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11986 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11987 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11988                          iRegIorL2I src1, iRegIorL2I src2,
11989                          immI src3, immI_M1 src4) %{
11990   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11991   ins_cost(1.9 * INSN_COST);
11992   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11993 
11994   ins_encode %{
11995     __ eonw(as_Register($dst$$reg),
11996               as_Register($src1$$reg),
11997               as_Register($src2$$reg),
11998               Assembler::LSR,
11999               $src3$$constant & 0x1f);
12000   %}
12001 
12002   ins_pipe(ialu_reg_reg_shift);
12003 %}
12004 
12005 // This pattern is automatically generated from aarch64_ad.m4
12006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12007 // val ^ (-1 ^ (val >>> shift)) ==> eon
12008 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
12009                          iRegL src1, iRegL src2,
12010                          immI src3, immL_M1 src4) %{
12011   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
12012   ins_cost(1.9 * INSN_COST);
12013   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
12014 
12015   ins_encode %{
12016     __ eon(as_Register($dst$$reg),
12017               as_Register($src1$$reg),
12018               as_Register($src2$$reg),
12019               Assembler::LSR,
12020               $src3$$constant & 0x3f);
12021   %}
12022 
12023   ins_pipe(ialu_reg_reg_shift);
12024 %}
12025 
12026 // This pattern is automatically generated from aarch64_ad.m4
12027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12028 // val ^ (-1 ^ (val >> shift)) ==> eonw
12029 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
12030                          iRegIorL2I src1, iRegIorL2I src2,
12031                          immI src3, immI_M1 src4) %{
12032   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
12033   ins_cost(1.9 * INSN_COST);
12034   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
12035 
12036   ins_encode %{
12037     __ eonw(as_Register($dst$$reg),
12038               as_Register($src1$$reg),
12039               as_Register($src2$$reg),
12040               Assembler::ASR,
12041               $src3$$constant & 0x1f);
12042   %}
12043 
12044   ins_pipe(ialu_reg_reg_shift);
12045 %}
12046 
12047 // This pattern is automatically generated from aarch64_ad.m4
12048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12049 // val ^ (-1 ^ (val >> shift)) ==> eon
12050 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
12051                          iRegL src1, iRegL src2,
12052                          immI src3, immL_M1 src4) %{
12053   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
12054   ins_cost(1.9 * INSN_COST);
12055   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
12056 
12057   ins_encode %{
12058     __ eon(as_Register($dst$$reg),
12059               as_Register($src1$$reg),
12060               as_Register($src2$$reg),
12061               Assembler::ASR,
12062               $src3$$constant & 0x3f);
12063   %}
12064 
12065   ins_pipe(ialu_reg_reg_shift);
12066 %}
12067 
12068 // This pattern is automatically generated from aarch64_ad.m4
12069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12070 // val ^ (-1 ^ (val ror shift)) ==> eonw
12071 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
12072                          iRegIorL2I src1, iRegIorL2I src2,
12073                          immI src3, immI_M1 src4) %{
12074   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
12075   ins_cost(1.9 * INSN_COST);
12076   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
12077 
12078   ins_encode %{
12079     __ eonw(as_Register($dst$$reg),
12080               as_Register($src1$$reg),
12081               as_Register($src2$$reg),
12082               Assembler::ROR,
12083               $src3$$constant & 0x1f);
12084   %}
12085 
12086   ins_pipe(ialu_reg_reg_shift);
12087 %}
12088 
12089 // This pattern is automatically generated from aarch64_ad.m4
12090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12091 // val ^ (-1 ^ (val ror shift)) ==> eon
12092 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
12093                          iRegL src1, iRegL src2,
12094                          immI src3, immL_M1 src4) %{
12095   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
12096   ins_cost(1.9 * INSN_COST);
12097   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
12098 
12099   ins_encode %{
12100     __ eon(as_Register($dst$$reg),
12101               as_Register($src1$$reg),
12102               as_Register($src2$$reg),
12103               Assembler::ROR,
12104               $src3$$constant & 0x3f);
12105   %}
12106 
12107   ins_pipe(ialu_reg_reg_shift);
12108 %}
12109 
12110 // This pattern is automatically generated from aarch64_ad.m4
12111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12112 // val ^ (-1 ^ (val << shift)) ==> eonw
12113 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
12114                          iRegIorL2I src1, iRegIorL2I src2,
12115                          immI src3, immI_M1 src4) %{
12116   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
12117   ins_cost(1.9 * INSN_COST);
12118   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
12119 
12120   ins_encode %{
12121     __ eonw(as_Register($dst$$reg),
12122               as_Register($src1$$reg),
12123               as_Register($src2$$reg),
12124               Assembler::LSL,
12125               $src3$$constant & 0x1f);
12126   %}
12127 
12128   ins_pipe(ialu_reg_reg_shift);
12129 %}
12130 
12131 // This pattern is automatically generated from aarch64_ad.m4
12132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12133 // val ^ (-1 ^ (val << shift)) ==> eon
12134 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
12135                          iRegL src1, iRegL src2,
12136                          immI src3, immL_M1 src4) %{
12137   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12138   ins_cost(1.9 * INSN_COST);
12139   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
12140 
12141   ins_encode %{
12142     __ eon(as_Register($dst$$reg),
12143               as_Register($src1$$reg),
12144               as_Register($src2$$reg),
12145               Assembler::LSL,
12146               $src3$$constant & 0x3f);
12147   %}
12148 
12149   ins_pipe(ialu_reg_reg_shift);
12150 %}
12151 
12152 // This pattern is automatically generated from aarch64_ad.m4
12153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12154 // val | (-1 ^ (val >>> shift)) ==> ornw
12155 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12156                          iRegIorL2I src1, iRegIorL2I src2,
12157                          immI src3, immI_M1 src4) %{
12158   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12159   ins_cost(1.9 * INSN_COST);
12160   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
12161 
12162   ins_encode %{
12163     __ ornw(as_Register($dst$$reg),
12164               as_Register($src1$$reg),
12165               as_Register($src2$$reg),
12166               Assembler::LSR,
12167               $src3$$constant & 0x1f);
12168   %}
12169 
12170   ins_pipe(ialu_reg_reg_shift);
12171 %}
12172 
12173 // This pattern is automatically generated from aarch64_ad.m4
12174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12175 // val | (-1 ^ (val >>> shift)) ==> orn
12176 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12177                          iRegL src1, iRegL src2,
12178                          immI src3, immL_M1 src4) %{
12179   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12180   ins_cost(1.9 * INSN_COST);
12181   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
12182 
12183   ins_encode %{
12184     __ orn(as_Register($dst$$reg),
12185               as_Register($src1$$reg),
12186               as_Register($src2$$reg),
12187               Assembler::LSR,
12188               $src3$$constant & 0x3f);
12189   %}
12190 
12191   ins_pipe(ialu_reg_reg_shift);
12192 %}
12193 
12194 // This pattern is automatically generated from aarch64_ad.m4
12195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12196 // val | (-1 ^ (val >> shift)) ==> ornw
12197 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12198                          iRegIorL2I src1, iRegIorL2I src2,
12199                          immI src3, immI_M1 src4) %{
12200   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12201   ins_cost(1.9 * INSN_COST);
12202   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12203 
12204   ins_encode %{
12205     __ ornw(as_Register($dst$$reg),
12206               as_Register($src1$$reg),
12207               as_Register($src2$$reg),
12208               Assembler::ASR,
12209               $src3$$constant & 0x1f);
12210   %}
12211 
12212   ins_pipe(ialu_reg_reg_shift);
12213 %}
12214 
12215 // This pattern is automatically generated from aarch64_ad.m4
12216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12217 // val | (-1 ^ (val >> shift)) ==> orn
12218 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12219                          iRegL src1, iRegL src2,
12220                          immI src3, immL_M1 src4) %{
12221   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12222   ins_cost(1.9 * INSN_COST);
12223   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12224 
12225   ins_encode %{
12226     __ orn(as_Register($dst$$reg),
12227               as_Register($src1$$reg),
12228               as_Register($src2$$reg),
12229               Assembler::ASR,
12230               $src3$$constant & 0x3f);
12231   %}
12232 
12233   ins_pipe(ialu_reg_reg_shift);
12234 %}
12235 
12236 // This pattern is automatically generated from aarch64_ad.m4
12237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12238 // val | (-1 ^ (val ror shift)) ==> ornw
12239 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12240                          iRegIorL2I src1, iRegIorL2I src2,
12241                          immI src3, immI_M1 src4) %{
12242   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12243   ins_cost(1.9 * INSN_COST);
12244   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12245 
12246   ins_encode %{
12247     __ ornw(as_Register($dst$$reg),
12248               as_Register($src1$$reg),
12249               as_Register($src2$$reg),
12250               Assembler::ROR,
12251               $src3$$constant & 0x1f);
12252   %}
12253 
12254   ins_pipe(ialu_reg_reg_shift);
12255 %}
12256 
12257 // This pattern is automatically generated from aarch64_ad.m4
12258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12259 // val | (-1 ^ (val ror shift)) ==> orn
12260 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12261                          iRegL src1, iRegL src2,
12262                          immI src3, immL_M1 src4) %{
12263   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12264   ins_cost(1.9 * INSN_COST);
12265   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12266 
12267   ins_encode %{
12268     __ orn(as_Register($dst$$reg),
12269               as_Register($src1$$reg),
12270               as_Register($src2$$reg),
12271               Assembler::ROR,
12272               $src3$$constant & 0x3f);
12273   %}
12274 
12275   ins_pipe(ialu_reg_reg_shift);
12276 %}
12277 
12278 // This pattern is automatically generated from aarch64_ad.m4
12279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12280 // val | (-1 ^ (val << shift)) ==> ornw
12281 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12282                          iRegIorL2I src1, iRegIorL2I src2,
12283                          immI src3, immI_M1 src4) %{
12284   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12285   ins_cost(1.9 * INSN_COST);
12286   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12287 
12288   ins_encode %{
12289     __ ornw(as_Register($dst$$reg),
12290               as_Register($src1$$reg),
12291               as_Register($src2$$reg),
12292               Assembler::LSL,
12293               $src3$$constant & 0x1f);
12294   %}
12295 
12296   ins_pipe(ialu_reg_reg_shift);
12297 %}
12298 
12299 // This pattern is automatically generated from aarch64_ad.m4
12300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12301 // val | (-1 ^ (val << shift)) ==> orn
12302 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12303                          iRegL src1, iRegL src2,
12304                          immI src3, immL_M1 src4) %{
12305   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12306   ins_cost(1.9 * INSN_COST);
12307   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12308 
12309   ins_encode %{
12310     __ orn(as_Register($dst$$reg),
12311               as_Register($src1$$reg),
12312               as_Register($src2$$reg),
12313               Assembler::LSL,
12314               $src3$$constant & 0x3f);
12315   %}
12316 
12317   ins_pipe(ialu_reg_reg_shift);
12318 %}
12319 
12320 // This pattern is automatically generated from aarch64_ad.m4
12321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12322 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12323                          iRegIorL2I src1, iRegIorL2I src2,
12324                          immI src3) %{
12325   match(Set dst (AndI src1 (URShiftI src2 src3)));
12326 
12327   ins_cost(1.9 * INSN_COST);
12328   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12329 
12330   ins_encode %{
12331     __ andw(as_Register($dst$$reg),
12332               as_Register($src1$$reg),
12333               as_Register($src2$$reg),
12334               Assembler::LSR,
12335               $src3$$constant & 0x1f);
12336   %}
12337 
12338   ins_pipe(ialu_reg_reg_shift);
12339 %}
12340 
12341 // This pattern is automatically generated from aarch64_ad.m4
12342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12343 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12344                          iRegL src1, iRegL src2,
12345                          immI src3) %{
12346   match(Set dst (AndL src1 (URShiftL src2 src3)));
12347 
12348   ins_cost(1.9 * INSN_COST);
12349   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12350 
12351   ins_encode %{
12352     __ andr(as_Register($dst$$reg),
12353               as_Register($src1$$reg),
12354               as_Register($src2$$reg),
12355               Assembler::LSR,
12356               $src3$$constant & 0x3f);
12357   %}
12358 
12359   ins_pipe(ialu_reg_reg_shift);
12360 %}
12361 
12362 // This pattern is automatically generated from aarch64_ad.m4
12363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12364 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12365                          iRegIorL2I src1, iRegIorL2I src2,
12366                          immI src3) %{
12367   match(Set dst (AndI src1 (RShiftI src2 src3)));
12368 
12369   ins_cost(1.9 * INSN_COST);
12370   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12371 
12372   ins_encode %{
12373     __ andw(as_Register($dst$$reg),
12374               as_Register($src1$$reg),
12375               as_Register($src2$$reg),
12376               Assembler::ASR,
12377               $src3$$constant & 0x1f);
12378   %}
12379 
12380   ins_pipe(ialu_reg_reg_shift);
12381 %}
12382 
12383 // This pattern is automatically generated from aarch64_ad.m4
12384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12385 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12386                          iRegL src1, iRegL src2,
12387                          immI src3) %{
12388   match(Set dst (AndL src1 (RShiftL src2 src3)));
12389 
12390   ins_cost(1.9 * INSN_COST);
12391   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12392 
12393   ins_encode %{
12394     __ andr(as_Register($dst$$reg),
12395               as_Register($src1$$reg),
12396               as_Register($src2$$reg),
12397               Assembler::ASR,
12398               $src3$$constant & 0x3f);
12399   %}
12400 
12401   ins_pipe(ialu_reg_reg_shift);
12402 %}
12403 
12404 // This pattern is automatically generated from aarch64_ad.m4
12405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12406 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12407                          iRegIorL2I src1, iRegIorL2I src2,
12408                          immI src3) %{
12409   match(Set dst (AndI src1 (LShiftI src2 src3)));
12410 
12411   ins_cost(1.9 * INSN_COST);
12412   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12413 
12414   ins_encode %{
12415     __ andw(as_Register($dst$$reg),
12416               as_Register($src1$$reg),
12417               as_Register($src2$$reg),
12418               Assembler::LSL,
12419               $src3$$constant & 0x1f);
12420   %}
12421 
12422   ins_pipe(ialu_reg_reg_shift);
12423 %}
12424 
12425 // This pattern is automatically generated from aarch64_ad.m4
12426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12427 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12428                          iRegL src1, iRegL src2,
12429                          immI src3) %{
12430   match(Set dst (AndL src1 (LShiftL src2 src3)));
12431 
12432   ins_cost(1.9 * INSN_COST);
12433   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12434 
12435   ins_encode %{
12436     __ andr(as_Register($dst$$reg),
12437               as_Register($src1$$reg),
12438               as_Register($src2$$reg),
12439               Assembler::LSL,
12440               $src3$$constant & 0x3f);
12441   %}
12442 
12443   ins_pipe(ialu_reg_reg_shift);
12444 %}
12445 
12446 // This pattern is automatically generated from aarch64_ad.m4
12447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12448 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12449                          iRegIorL2I src1, iRegIorL2I src2,
12450                          immI src3) %{
12451   match(Set dst (AndI src1 (RotateRight src2 src3)));
12452 
12453   ins_cost(1.9 * INSN_COST);
12454   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12455 
12456   ins_encode %{
12457     __ andw(as_Register($dst$$reg),
12458               as_Register($src1$$reg),
12459               as_Register($src2$$reg),
12460               Assembler::ROR,
12461               $src3$$constant & 0x1f);
12462   %}
12463 
12464   ins_pipe(ialu_reg_reg_shift);
12465 %}
12466 
12467 // This pattern is automatically generated from aarch64_ad.m4
12468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12469 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12470                          iRegL src1, iRegL src2,
12471                          immI src3) %{
12472   match(Set dst (AndL src1 (RotateRight src2 src3)));
12473 
12474   ins_cost(1.9 * INSN_COST);
12475   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12476 
12477   ins_encode %{
12478     __ andr(as_Register($dst$$reg),
12479               as_Register($src1$$reg),
12480               as_Register($src2$$reg),
12481               Assembler::ROR,
12482               $src3$$constant & 0x3f);
12483   %}
12484 
12485   ins_pipe(ialu_reg_reg_shift);
12486 %}
12487 
12488 // This pattern is automatically generated from aarch64_ad.m4
12489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12490 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12491                          iRegIorL2I src1, iRegIorL2I src2,
12492                          immI src3) %{
12493   match(Set dst (XorI src1 (URShiftI src2 src3)));
12494 
12495   ins_cost(1.9 * INSN_COST);
12496   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12497 
12498   ins_encode %{
12499     __ eorw(as_Register($dst$$reg),
12500               as_Register($src1$$reg),
12501               as_Register($src2$$reg),
12502               Assembler::LSR,
12503               $src3$$constant & 0x1f);
12504   %}
12505 
12506   ins_pipe(ialu_reg_reg_shift);
12507 %}
12508 
12509 // This pattern is automatically generated from aarch64_ad.m4
12510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12511 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12512                          iRegL src1, iRegL src2,
12513                          immI src3) %{
12514   match(Set dst (XorL src1 (URShiftL src2 src3)));
12515 
12516   ins_cost(1.9 * INSN_COST);
12517   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12518 
12519   ins_encode %{
12520     __ eor(as_Register($dst$$reg),
12521               as_Register($src1$$reg),
12522               as_Register($src2$$reg),
12523               Assembler::LSR,
12524               $src3$$constant & 0x3f);
12525   %}
12526 
12527   ins_pipe(ialu_reg_reg_shift);
12528 %}
12529 
12530 // This pattern is automatically generated from aarch64_ad.m4
12531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12532 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12533                          iRegIorL2I src1, iRegIorL2I src2,
12534                          immI src3) %{
12535   match(Set dst (XorI src1 (RShiftI src2 src3)));
12536 
12537   ins_cost(1.9 * INSN_COST);
12538   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12539 
12540   ins_encode %{
12541     __ eorw(as_Register($dst$$reg),
12542               as_Register($src1$$reg),
12543               as_Register($src2$$reg),
12544               Assembler::ASR,
12545               $src3$$constant & 0x1f);
12546   %}
12547 
12548   ins_pipe(ialu_reg_reg_shift);
12549 %}
12550 
12551 // This pattern is automatically generated from aarch64_ad.m4
12552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12553 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12554                          iRegL src1, iRegL src2,
12555                          immI src3) %{
12556   match(Set dst (XorL src1 (RShiftL src2 src3)));
12557 
12558   ins_cost(1.9 * INSN_COST);
12559   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12560 
12561   ins_encode %{
12562     __ eor(as_Register($dst$$reg),
12563               as_Register($src1$$reg),
12564               as_Register($src2$$reg),
12565               Assembler::ASR,
12566               $src3$$constant & 0x3f);
12567   %}
12568 
12569   ins_pipe(ialu_reg_reg_shift);
12570 %}
12571 
12572 // This pattern is automatically generated from aarch64_ad.m4
12573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12574 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12575                          iRegIorL2I src1, iRegIorL2I src2,
12576                          immI src3) %{
12577   match(Set dst (XorI src1 (LShiftI src2 src3)));
12578 
12579   ins_cost(1.9 * INSN_COST);
12580   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12581 
12582   ins_encode %{
12583     __ eorw(as_Register($dst$$reg),
12584               as_Register($src1$$reg),
12585               as_Register($src2$$reg),
12586               Assembler::LSL,
12587               $src3$$constant & 0x1f);
12588   %}
12589 
12590   ins_pipe(ialu_reg_reg_shift);
12591 %}
12592 
12593 // This pattern is automatically generated from aarch64_ad.m4
12594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12595 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12596                          iRegL src1, iRegL src2,
12597                          immI src3) %{
12598   match(Set dst (XorL src1 (LShiftL src2 src3)));
12599 
12600   ins_cost(1.9 * INSN_COST);
12601   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12602 
12603   ins_encode %{
12604     __ eor(as_Register($dst$$reg),
12605               as_Register($src1$$reg),
12606               as_Register($src2$$reg),
12607               Assembler::LSL,
12608               $src3$$constant & 0x3f);
12609   %}
12610 
12611   ins_pipe(ialu_reg_reg_shift);
12612 %}
12613 
12614 // This pattern is automatically generated from aarch64_ad.m4
12615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12616 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12617                          iRegIorL2I src1, iRegIorL2I src2,
12618                          immI src3) %{
12619   match(Set dst (XorI src1 (RotateRight src2 src3)));
12620 
12621   ins_cost(1.9 * INSN_COST);
12622   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12623 
12624   ins_encode %{
12625     __ eorw(as_Register($dst$$reg),
12626               as_Register($src1$$reg),
12627               as_Register($src2$$reg),
12628               Assembler::ROR,
12629               $src3$$constant & 0x1f);
12630   %}
12631 
12632   ins_pipe(ialu_reg_reg_shift);
12633 %}
12634 
12635 // This pattern is automatically generated from aarch64_ad.m4
12636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12637 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12638                          iRegL src1, iRegL src2,
12639                          immI src3) %{
12640   match(Set dst (XorL src1 (RotateRight src2 src3)));
12641 
12642   ins_cost(1.9 * INSN_COST);
12643   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12644 
12645   ins_encode %{
12646     __ eor(as_Register($dst$$reg),
12647               as_Register($src1$$reg),
12648               as_Register($src2$$reg),
12649               Assembler::ROR,
12650               $src3$$constant & 0x3f);
12651   %}
12652 
12653   ins_pipe(ialu_reg_reg_shift);
12654 %}
12655 
12656 // This pattern is automatically generated from aarch64_ad.m4
12657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12658 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12659                          iRegIorL2I src1, iRegIorL2I src2,
12660                          immI src3) %{
12661   match(Set dst (OrI src1 (URShiftI src2 src3)));
12662 
12663   ins_cost(1.9 * INSN_COST);
12664   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12665 
12666   ins_encode %{
12667     __ orrw(as_Register($dst$$reg),
12668               as_Register($src1$$reg),
12669               as_Register($src2$$reg),
12670               Assembler::LSR,
12671               $src3$$constant & 0x1f);
12672   %}
12673 
12674   ins_pipe(ialu_reg_reg_shift);
12675 %}
12676 
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12680                          iRegL src1, iRegL src2,
12681                          immI src3) %{
12682   match(Set dst (OrL src1 (URShiftL src2 src3)));
12683 
12684   ins_cost(1.9 * INSN_COST);
12685   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12686 
12687   ins_encode %{
12688     __ orr(as_Register($dst$$reg),
12689               as_Register($src1$$reg),
12690               as_Register($src2$$reg),
12691               Assembler::LSR,
12692               $src3$$constant & 0x3f);
12693   %}
12694 
12695   ins_pipe(ialu_reg_reg_shift);
12696 %}
12697 
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12701                          iRegIorL2I src1, iRegIorL2I src2,
12702                          immI src3) %{
12703   match(Set dst (OrI src1 (RShiftI src2 src3)));
12704 
12705   ins_cost(1.9 * INSN_COST);
12706   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12707 
12708   ins_encode %{
12709     __ orrw(as_Register($dst$$reg),
12710               as_Register($src1$$reg),
12711               as_Register($src2$$reg),
12712               Assembler::ASR,
12713               $src3$$constant & 0x1f);
12714   %}
12715 
12716   ins_pipe(ialu_reg_reg_shift);
12717 %}
12718 
12719 // This pattern is automatically generated from aarch64_ad.m4
12720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12721 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12722                          iRegL src1, iRegL src2,
12723                          immI src3) %{
12724   match(Set dst (OrL src1 (RShiftL src2 src3)));
12725 
12726   ins_cost(1.9 * INSN_COST);
12727   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12728 
12729   ins_encode %{
12730     __ orr(as_Register($dst$$reg),
12731               as_Register($src1$$reg),
12732               as_Register($src2$$reg),
12733               Assembler::ASR,
12734               $src3$$constant & 0x3f);
12735   %}
12736 
12737   ins_pipe(ialu_reg_reg_shift);
12738 %}
12739 
12740 // This pattern is automatically generated from aarch64_ad.m4
12741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12742 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12743                          iRegIorL2I src1, iRegIorL2I src2,
12744                          immI src3) %{
12745   match(Set dst (OrI src1 (LShiftI src2 src3)));
12746 
12747   ins_cost(1.9 * INSN_COST);
12748   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12749 
12750   ins_encode %{
12751     __ orrw(as_Register($dst$$reg),
12752               as_Register($src1$$reg),
12753               as_Register($src2$$reg),
12754               Assembler::LSL,
12755               $src3$$constant & 0x1f);
12756   %}
12757 
12758   ins_pipe(ialu_reg_reg_shift);
12759 %}
12760 
12761 // This pattern is automatically generated from aarch64_ad.m4
12762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12763 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12764                          iRegL src1, iRegL src2,
12765                          immI src3) %{
12766   match(Set dst (OrL src1 (LShiftL src2 src3)));
12767 
12768   ins_cost(1.9 * INSN_COST);
12769   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12770 
12771   ins_encode %{
12772     __ orr(as_Register($dst$$reg),
12773               as_Register($src1$$reg),
12774               as_Register($src2$$reg),
12775               Assembler::LSL,
12776               $src3$$constant & 0x3f);
12777   %}
12778 
12779   ins_pipe(ialu_reg_reg_shift);
12780 %}
12781 
12782 // This pattern is automatically generated from aarch64_ad.m4
12783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12784 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12785                          iRegIorL2I src1, iRegIorL2I src2,
12786                          immI src3) %{
12787   match(Set dst (OrI src1 (RotateRight src2 src3)));
12788 
12789   ins_cost(1.9 * INSN_COST);
12790   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12791 
12792   ins_encode %{
12793     __ orrw(as_Register($dst$$reg),
12794               as_Register($src1$$reg),
12795               as_Register($src2$$reg),
12796               Assembler::ROR,
12797               $src3$$constant & 0x1f);
12798   %}
12799 
12800   ins_pipe(ialu_reg_reg_shift);
12801 %}
12802 
12803 // This pattern is automatically generated from aarch64_ad.m4
12804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12805 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12806                          iRegL src1, iRegL src2,
12807                          immI src3) %{
12808   match(Set dst (OrL src1 (RotateRight src2 src3)));
12809 
12810   ins_cost(1.9 * INSN_COST);
12811   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12812 
12813   ins_encode %{
12814     __ orr(as_Register($dst$$reg),
12815               as_Register($src1$$reg),
12816               as_Register($src2$$reg),
12817               Assembler::ROR,
12818               $src3$$constant & 0x3f);
12819   %}
12820 
12821   ins_pipe(ialu_reg_reg_shift);
12822 %}
12823 
12824 // This pattern is automatically generated from aarch64_ad.m4
12825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12826 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12827                          iRegIorL2I src1, iRegIorL2I src2,
12828                          immI src3) %{
12829   match(Set dst (AddI src1 (URShiftI src2 src3)));
12830 
12831   ins_cost(1.9 * INSN_COST);
12832   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12833 
12834   ins_encode %{
12835     __ addw(as_Register($dst$$reg),
12836               as_Register($src1$$reg),
12837               as_Register($src2$$reg),
12838               Assembler::LSR,
12839               $src3$$constant & 0x1f);
12840   %}
12841 
12842   ins_pipe(ialu_reg_reg_shift);
12843 %}
12844 
12845 // This pattern is automatically generated from aarch64_ad.m4
12846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12847 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12848                          iRegL src1, iRegL src2,
12849                          immI src3) %{
12850   match(Set dst (AddL src1 (URShiftL src2 src3)));
12851 
12852   ins_cost(1.9 * INSN_COST);
12853   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12854 
12855   ins_encode %{
12856     __ add(as_Register($dst$$reg),
12857               as_Register($src1$$reg),
12858               as_Register($src2$$reg),
12859               Assembler::LSR,
12860               $src3$$constant & 0x3f);
12861   %}
12862 
12863   ins_pipe(ialu_reg_reg_shift);
12864 %}
12865 
12866 // This pattern is automatically generated from aarch64_ad.m4
12867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12868 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12869                          iRegIorL2I src1, iRegIorL2I src2,
12870                          immI src3) %{
12871   match(Set dst (AddI src1 (RShiftI src2 src3)));
12872 
12873   ins_cost(1.9 * INSN_COST);
12874   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12875 
12876   ins_encode %{
12877     __ addw(as_Register($dst$$reg),
12878               as_Register($src1$$reg),
12879               as_Register($src2$$reg),
12880               Assembler::ASR,
12881               $src3$$constant & 0x1f);
12882   %}
12883 
12884   ins_pipe(ialu_reg_reg_shift);
12885 %}
12886 
12887 // This pattern is automatically generated from aarch64_ad.m4
12888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12889 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12890                          iRegL src1, iRegL src2,
12891                          immI src3) %{
12892   match(Set dst (AddL src1 (RShiftL src2 src3)));
12893 
12894   ins_cost(1.9 * INSN_COST);
12895   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12896 
12897   ins_encode %{
12898     __ add(as_Register($dst$$reg),
12899               as_Register($src1$$reg),
12900               as_Register($src2$$reg),
12901               Assembler::ASR,
12902               $src3$$constant & 0x3f);
12903   %}
12904 
12905   ins_pipe(ialu_reg_reg_shift);
12906 %}
12907 
12908 // This pattern is automatically generated from aarch64_ad.m4
12909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12910 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12911                          iRegIorL2I src1, iRegIorL2I src2,
12912                          immI src3) %{
12913   match(Set dst (AddI src1 (LShiftI src2 src3)));
12914 
12915   ins_cost(1.9 * INSN_COST);
12916   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12917 
12918   ins_encode %{
12919     __ addw(as_Register($dst$$reg),
12920               as_Register($src1$$reg),
12921               as_Register($src2$$reg),
12922               Assembler::LSL,
12923               $src3$$constant & 0x1f);
12924   %}
12925 
12926   ins_pipe(ialu_reg_reg_shift);
12927 %}
12928 
12929 // This pattern is automatically generated from aarch64_ad.m4
12930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12931 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12932                          iRegL src1, iRegL src2,
12933                          immI src3) %{
12934   match(Set dst (AddL src1 (LShiftL src2 src3)));
12935 
12936   ins_cost(1.9 * INSN_COST);
12937   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12938 
12939   ins_encode %{
12940     __ add(as_Register($dst$$reg),
12941               as_Register($src1$$reg),
12942               as_Register($src2$$reg),
12943               Assembler::LSL,
12944               $src3$$constant & 0x3f);
12945   %}
12946 
12947   ins_pipe(ialu_reg_reg_shift);
12948 %}
12949 
12950 // This pattern is automatically generated from aarch64_ad.m4
12951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12952 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12953                          iRegIorL2I src1, iRegIorL2I src2,
12954                          immI src3) %{
12955   match(Set dst (SubI src1 (URShiftI src2 src3)));
12956 
12957   ins_cost(1.9 * INSN_COST);
12958   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12959 
12960   ins_encode %{
12961     __ subw(as_Register($dst$$reg),
12962               as_Register($src1$$reg),
12963               as_Register($src2$$reg),
12964               Assembler::LSR,
12965               $src3$$constant & 0x1f);
12966   %}
12967 
12968   ins_pipe(ialu_reg_reg_shift);
12969 %}
12970 
12971 // This pattern is automatically generated from aarch64_ad.m4
12972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12973 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12974                          iRegL src1, iRegL src2,
12975                          immI src3) %{
12976   match(Set dst (SubL src1 (URShiftL src2 src3)));
12977 
12978   ins_cost(1.9 * INSN_COST);
12979   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12980 
12981   ins_encode %{
12982     __ sub(as_Register($dst$$reg),
12983               as_Register($src1$$reg),
12984               as_Register($src2$$reg),
12985               Assembler::LSR,
12986               $src3$$constant & 0x3f);
12987   %}
12988 
12989   ins_pipe(ialu_reg_reg_shift);
12990 %}
12991 
12992 // This pattern is automatically generated from aarch64_ad.m4
12993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12994 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12995                          iRegIorL2I src1, iRegIorL2I src2,
12996                          immI src3) %{
12997   match(Set dst (SubI src1 (RShiftI src2 src3)));
12998 
12999   ins_cost(1.9 * INSN_COST);
13000   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
13001 
13002   ins_encode %{
13003     __ subw(as_Register($dst$$reg),
13004               as_Register($src1$$reg),
13005               as_Register($src2$$reg),
13006               Assembler::ASR,
13007               $src3$$constant & 0x1f);
13008   %}
13009 
13010   ins_pipe(ialu_reg_reg_shift);
13011 %}
13012 
13013 // This pattern is automatically generated from aarch64_ad.m4
13014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13015 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
13016                          iRegL src1, iRegL src2,
13017                          immI src3) %{
13018   match(Set dst (SubL src1 (RShiftL src2 src3)));
13019 
13020   ins_cost(1.9 * INSN_COST);
13021   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
13022 
13023   ins_encode %{
13024     __ sub(as_Register($dst$$reg),
13025               as_Register($src1$$reg),
13026               as_Register($src2$$reg),
13027               Assembler::ASR,
13028               $src3$$constant & 0x3f);
13029   %}
13030 
13031   ins_pipe(ialu_reg_reg_shift);
13032 %}
13033 
13034 // This pattern is automatically generated from aarch64_ad.m4
13035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13036 instruct SubI_reg_LShift_reg(iRegINoSp dst,
13037                          iRegIorL2I src1, iRegIorL2I src2,
13038                          immI src3) %{
13039   match(Set dst (SubI src1 (LShiftI src2 src3)));
13040 
13041   ins_cost(1.9 * INSN_COST);
13042   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
13043 
13044   ins_encode %{
13045     __ subw(as_Register($dst$$reg),
13046               as_Register($src1$$reg),
13047               as_Register($src2$$reg),
13048               Assembler::LSL,
13049               $src3$$constant & 0x1f);
13050   %}
13051 
13052   ins_pipe(ialu_reg_reg_shift);
13053 %}
13054 
13055 // This pattern is automatically generated from aarch64_ad.m4
13056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13057 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
13058                          iRegL src1, iRegL src2,
13059                          immI src3) %{
13060   match(Set dst (SubL src1 (LShiftL src2 src3)));
13061 
13062   ins_cost(1.9 * INSN_COST);
13063   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
13064 
13065   ins_encode %{
13066     __ sub(as_Register($dst$$reg),
13067               as_Register($src1$$reg),
13068               as_Register($src2$$reg),
13069               Assembler::LSL,
13070               $src3$$constant & 0x3f);
13071   %}
13072 
13073   ins_pipe(ialu_reg_reg_shift);
13074 %}
13075 
13076 // This pattern is automatically generated from aarch64_ad.m4
13077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13078 
13079 // Shift Left followed by Shift Right.
13080 // This idiom is used by the compiler for the i2b bytecode etc.
13081 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13082 %{
13083   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
13084   ins_cost(INSN_COST * 2);
13085   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13086   ins_encode %{
13087     int lshift = $lshift_count$$constant & 63;
13088     int rshift = $rshift_count$$constant & 63;
13089     int s = 63 - lshift;
13090     int r = (rshift - lshift) & 63;
13091     __ sbfm(as_Register($dst$$reg),
13092             as_Register($src$$reg),
13093             r, s);
13094   %}
13095 
13096   ins_pipe(ialu_reg_shift);
13097 %}
13098 
13099 // This pattern is automatically generated from aarch64_ad.m4
13100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13101 
13102 // Shift Left followed by Shift Right.
13103 // This idiom is used by the compiler for the i2b bytecode etc.
13104 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13105 %{
13106   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
13107   ins_cost(INSN_COST * 2);
13108   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13109   ins_encode %{
13110     int lshift = $lshift_count$$constant & 31;
13111     int rshift = $rshift_count$$constant & 31;
13112     int s = 31 - lshift;
13113     int r = (rshift - lshift) & 31;
13114     __ sbfmw(as_Register($dst$$reg),
13115             as_Register($src$$reg),
13116             r, s);
13117   %}
13118 
13119   ins_pipe(ialu_reg_shift);
13120 %}
13121 
13122 // This pattern is automatically generated from aarch64_ad.m4
13123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13124 
13125 // Shift Left followed by Shift Right.
13126 // This idiom is used by the compiler for the i2b bytecode etc.
13127 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13128 %{
13129   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
13130   ins_cost(INSN_COST * 2);
13131   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13132   ins_encode %{
13133     int lshift = $lshift_count$$constant & 63;
13134     int rshift = $rshift_count$$constant & 63;
13135     int s = 63 - lshift;
13136     int r = (rshift - lshift) & 63;
13137     __ ubfm(as_Register($dst$$reg),
13138             as_Register($src$$reg),
13139             r, s);
13140   %}
13141 
13142   ins_pipe(ialu_reg_shift);
13143 %}
13144 
13145 // This pattern is automatically generated from aarch64_ad.m4
13146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13147 
13148 // Shift Left followed by Shift Right.
13149 // This idiom is used by the compiler for the i2b bytecode etc.
13150 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13151 %{
13152   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13153   ins_cost(INSN_COST * 2);
13154   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13155   ins_encode %{
13156     int lshift = $lshift_count$$constant & 31;
13157     int rshift = $rshift_count$$constant & 31;
13158     int s = 31 - lshift;
13159     int r = (rshift - lshift) & 31;
13160     __ ubfmw(as_Register($dst$$reg),
13161             as_Register($src$$reg),
13162             r, s);
13163   %}
13164 
13165   ins_pipe(ialu_reg_shift);
13166 %}
13167 
13168 // Bitfield extract with shift & mask
13169 
13170 // This pattern is automatically generated from aarch64_ad.m4
13171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13172 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13173 %{
13174   match(Set dst (AndI (URShiftI src rshift) mask));
13175   // Make sure we are not going to exceed what ubfxw can do.
13176   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13177 
13178   ins_cost(INSN_COST);
13179   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13180   ins_encode %{
13181     int rshift = $rshift$$constant & 31;
13182     intptr_t mask = $mask$$constant;
13183     int width = exact_log2(mask+1);
13184     __ ubfxw(as_Register($dst$$reg),
13185             as_Register($src$$reg), rshift, width);
13186   %}
13187   ins_pipe(ialu_reg_shift);
13188 %}
13189 
13190 // This pattern is automatically generated from aarch64_ad.m4
13191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13192 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13193 %{
13194   match(Set dst (AndL (URShiftL src rshift) mask));
13195   // Make sure we are not going to exceed what ubfx can do.
13196   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13197 
13198   ins_cost(INSN_COST);
13199   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13200   ins_encode %{
13201     int rshift = $rshift$$constant & 63;
13202     intptr_t mask = $mask$$constant;
13203     int width = exact_log2_long(mask+1);
13204     __ ubfx(as_Register($dst$$reg),
13205             as_Register($src$$reg), rshift, width);
13206   %}
13207   ins_pipe(ialu_reg_shift);
13208 %}
13209 
13210 
13211 // This pattern is automatically generated from aarch64_ad.m4
13212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13213 
13214 // We can use ubfx when extending an And with a mask when we know mask
13215 // is positive.  We know that because immI_bitmask guarantees it.
13216 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13217 %{
13218   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13219   // Make sure we are not going to exceed what ubfxw can do.
13220   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13221 
13222   ins_cost(INSN_COST * 2);
13223   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13224   ins_encode %{
13225     int rshift = $rshift$$constant & 31;
13226     intptr_t mask = $mask$$constant;
13227     int width = exact_log2(mask+1);
13228     __ ubfx(as_Register($dst$$reg),
13229             as_Register($src$$reg), rshift, width);
13230   %}
13231   ins_pipe(ialu_reg_shift);
13232 %}
13233 
13234 
13235 // This pattern is automatically generated from aarch64_ad.m4
13236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13237 
13238 // We can use ubfiz when masking by a positive number and then left shifting the result.
13239 // We know that the mask is positive because immI_bitmask guarantees it.
13240 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13241 %{
13242   match(Set dst (LShiftI (AndI src mask) lshift));
13243   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13244 
13245   ins_cost(INSN_COST);
13246   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13247   ins_encode %{
13248     int lshift = $lshift$$constant & 31;
13249     intptr_t mask = $mask$$constant;
13250     int width = exact_log2(mask+1);
13251     __ ubfizw(as_Register($dst$$reg),
13252           as_Register($src$$reg), lshift, width);
13253   %}
13254   ins_pipe(ialu_reg_shift);
13255 %}
13256 
13257 // This pattern is automatically generated from aarch64_ad.m4
13258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13259 
13260 // We can use ubfiz when masking by a positive number and then left shifting the result.
13261 // We know that the mask is positive because immL_bitmask guarantees it.
13262 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13263 %{
13264   match(Set dst (LShiftL (AndL src mask) lshift));
13265   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13266 
13267   ins_cost(INSN_COST);
13268   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13269   ins_encode %{
13270     int lshift = $lshift$$constant & 63;
13271     intptr_t mask = $mask$$constant;
13272     int width = exact_log2_long(mask+1);
13273     __ ubfiz(as_Register($dst$$reg),
13274           as_Register($src$$reg), lshift, width);
13275   %}
13276   ins_pipe(ialu_reg_shift);
13277 %}
13278 
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 
13282 // We can use ubfiz when masking by a positive number and then left shifting the result.
13283 // We know that the mask is positive because immI_bitmask guarantees it.
13284 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13285 %{
13286   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13287   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13288 
13289   ins_cost(INSN_COST);
13290   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13291   ins_encode %{
13292     int lshift = $lshift$$constant & 31;
13293     intptr_t mask = $mask$$constant;
13294     int width = exact_log2(mask+1);
13295     __ ubfizw(as_Register($dst$$reg),
13296           as_Register($src$$reg), lshift, width);
13297   %}
13298   ins_pipe(ialu_reg_shift);
13299 %}
13300 
13301 // This pattern is automatically generated from aarch64_ad.m4
13302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13303 
13304 // We can use ubfiz when masking by a positive number and then left shifting the result.
13305 // We know that the mask is positive because immL_bitmask guarantees it.
13306 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13307 %{
13308   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13309   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13310 
13311   ins_cost(INSN_COST);
13312   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13313   ins_encode %{
13314     int lshift = $lshift$$constant & 63;
13315     intptr_t mask = $mask$$constant;
13316     int width = exact_log2_long(mask+1);
13317     __ ubfiz(as_Register($dst$$reg),
13318           as_Register($src$$reg), lshift, width);
13319   %}
13320   ins_pipe(ialu_reg_shift);
13321 %}
13322 
13323 
13324 // This pattern is automatically generated from aarch64_ad.m4
13325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13326 
13327 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13328 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13329 %{
13330   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13331   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13332 
13333   ins_cost(INSN_COST);
13334   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13335   ins_encode %{
13336     int lshift = $lshift$$constant & 63;
13337     intptr_t mask = $mask$$constant;
13338     int width = exact_log2(mask+1);
13339     __ ubfiz(as_Register($dst$$reg),
13340              as_Register($src$$reg), lshift, width);
13341   %}
13342   ins_pipe(ialu_reg_shift);
13343 %}
13344 
13345 // This pattern is automatically generated from aarch64_ad.m4
13346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13347 
13348 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13349 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13350 %{
13351   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13352   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13353 
13354   ins_cost(INSN_COST);
13355   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13356   ins_encode %{
13357     int lshift = $lshift$$constant & 31;
13358     intptr_t mask = $mask$$constant;
13359     int width = exact_log2(mask+1);
13360     __ ubfiz(as_Register($dst$$reg),
13361              as_Register($src$$reg), lshift, width);
13362   %}
13363   ins_pipe(ialu_reg_shift);
13364 %}
13365 
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 
13369 // Can skip int2long conversions after AND with small bitmask
13370 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13371 %{
13372   match(Set dst (ConvI2L (AndI src msk)));
13373   ins_cost(INSN_COST);
13374   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13375   ins_encode %{
13376     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13377   %}
13378   ins_pipe(ialu_reg_shift);
13379 %}
13380 
13381 
13382 // Rotations
13383 // This pattern is automatically generated from aarch64_ad.m4
13384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13385 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13386 %{
13387   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13388   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13389 
13390   ins_cost(INSN_COST);
13391   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13392 
13393   ins_encode %{
13394     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13395             $rshift$$constant & 63);
13396   %}
13397   ins_pipe(ialu_reg_reg_extr);
13398 %}
13399 
13400 
13401 // This pattern is automatically generated from aarch64_ad.m4
13402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13403 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13404 %{
13405   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13406   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13407 
13408   ins_cost(INSN_COST);
13409   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13410 
13411   ins_encode %{
13412     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13413             $rshift$$constant & 31);
13414   %}
13415   ins_pipe(ialu_reg_reg_extr);
13416 %}
13417 
13418 
13419 // This pattern is automatically generated from aarch64_ad.m4
13420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13421 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13422 %{
13423   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13424   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13425 
13426   ins_cost(INSN_COST);
13427   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13428 
13429   ins_encode %{
13430     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13431             $rshift$$constant & 63);
13432   %}
13433   ins_pipe(ialu_reg_reg_extr);
13434 %}
13435 
13436 
13437 // This pattern is automatically generated from aarch64_ad.m4
13438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13439 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13440 %{
13441   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13442   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13443 
13444   ins_cost(INSN_COST);
13445   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13446 
13447   ins_encode %{
13448     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13449             $rshift$$constant & 31);
13450   %}
13451   ins_pipe(ialu_reg_reg_extr);
13452 %}
13453 
13454 
13455 // This pattern is automatically generated from aarch64_ad.m4
13456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13457 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13458 %{
13459   match(Set dst (RotateRight src shift));
13460 
13461   ins_cost(INSN_COST);
13462   format %{ "ror    $dst, $src, $shift" %}
13463 
13464   ins_encode %{
13465      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13466                $shift$$constant & 0x1f);
13467   %}
13468   ins_pipe(ialu_reg_reg_vshift);
13469 %}
13470 
13471 // This pattern is automatically generated from aarch64_ad.m4
13472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13473 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13474 %{
13475   match(Set dst (RotateRight src shift));
13476 
13477   ins_cost(INSN_COST);
13478   format %{ "ror    $dst, $src, $shift" %}
13479 
13480   ins_encode %{
13481      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13482                $shift$$constant & 0x3f);
13483   %}
13484   ins_pipe(ialu_reg_reg_vshift);
13485 %}
13486 
13487 // This pattern is automatically generated from aarch64_ad.m4
13488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13489 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13490 %{
13491   match(Set dst (RotateRight src shift));
13492 
13493   ins_cost(INSN_COST);
13494   format %{ "ror    $dst, $src, $shift" %}
13495 
13496   ins_encode %{
13497      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13498   %}
13499   ins_pipe(ialu_reg_reg_vshift);
13500 %}
13501 
13502 // This pattern is automatically generated from aarch64_ad.m4
13503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13504 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13505 %{
13506   match(Set dst (RotateRight src shift));
13507 
13508   ins_cost(INSN_COST);
13509   format %{ "ror    $dst, $src, $shift" %}
13510 
13511   ins_encode %{
13512      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13513   %}
13514   ins_pipe(ialu_reg_reg_vshift);
13515 %}
13516 
13517 // This pattern is automatically generated from aarch64_ad.m4
13518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13519 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13520 %{
13521   match(Set dst (RotateLeft src shift));
13522 
13523   ins_cost(INSN_COST);
13524   format %{ "rol    $dst, $src, $shift" %}
13525 
13526   ins_encode %{
13527      __ subw(rscratch1, zr, as_Register($shift$$reg));
13528      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13529   %}
13530   ins_pipe(ialu_reg_reg_vshift);
13531 %}
13532 
13533 // This pattern is automatically generated from aarch64_ad.m4
13534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13535 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13536 %{
13537   match(Set dst (RotateLeft src shift));
13538 
13539   ins_cost(INSN_COST);
13540   format %{ "rol    $dst, $src, $shift" %}
13541 
13542   ins_encode %{
13543      __ subw(rscratch1, zr, as_Register($shift$$reg));
13544      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13545   %}
13546   ins_pipe(ialu_reg_reg_vshift);
13547 %}
13548 
13549 
13550 // Add/subtract (extended)
13551 
13552 // This pattern is automatically generated from aarch64_ad.m4
13553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13554 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13555 %{
13556   match(Set dst (AddL src1 (ConvI2L src2)));
13557   ins_cost(INSN_COST);
13558   format %{ "add  $dst, $src1, $src2, sxtw" %}
13559 
13560    ins_encode %{
13561      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13562             as_Register($src2$$reg), ext::sxtw);
13563    %}
13564   ins_pipe(ialu_reg_reg);
13565 %}
13566 
13567 // This pattern is automatically generated from aarch64_ad.m4
13568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13569 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13570 %{
13571   match(Set dst (SubL src1 (ConvI2L src2)));
13572   ins_cost(INSN_COST);
13573   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13574 
13575    ins_encode %{
13576      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13577             as_Register($src2$$reg), ext::sxtw);
13578    %}
13579   ins_pipe(ialu_reg_reg);
13580 %}
13581 
13582 // This pattern is automatically generated from aarch64_ad.m4
13583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13584 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13585 %{
13586   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13587   ins_cost(INSN_COST);
13588   format %{ "add  $dst, $src1, $src2, sxth" %}
13589 
13590    ins_encode %{
13591      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13592             as_Register($src2$$reg), ext::sxth);
13593    %}
13594   ins_pipe(ialu_reg_reg);
13595 %}
13596 
13597 // This pattern is automatically generated from aarch64_ad.m4
13598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13599 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13600 %{
13601   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13602   ins_cost(INSN_COST);
13603   format %{ "add  $dst, $src1, $src2, sxtb" %}
13604 
13605    ins_encode %{
13606      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13607             as_Register($src2$$reg), ext::sxtb);
13608    %}
13609   ins_pipe(ialu_reg_reg);
13610 %}
13611 
13612 // This pattern is automatically generated from aarch64_ad.m4
13613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13614 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13615 %{
13616   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13617   ins_cost(INSN_COST);
13618   format %{ "add  $dst, $src1, $src2, uxtb" %}
13619 
13620    ins_encode %{
13621      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13622             as_Register($src2$$reg), ext::uxtb);
13623    %}
13624   ins_pipe(ialu_reg_reg);
13625 %}
13626 
13627 // This pattern is automatically generated from aarch64_ad.m4
13628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13629 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13630 %{
13631   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13632   ins_cost(INSN_COST);
13633   format %{ "add  $dst, $src1, $src2, sxth" %}
13634 
13635    ins_encode %{
13636      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13637             as_Register($src2$$reg), ext::sxth);
13638    %}
13639   ins_pipe(ialu_reg_reg);
13640 %}
13641 
13642 // This pattern is automatically generated from aarch64_ad.m4
13643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13644 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13645 %{
13646   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13647   ins_cost(INSN_COST);
13648   format %{ "add  $dst, $src1, $src2, sxtw" %}
13649 
13650    ins_encode %{
13651      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13652             as_Register($src2$$reg), ext::sxtw);
13653    %}
13654   ins_pipe(ialu_reg_reg);
13655 %}
13656 
13657 // This pattern is automatically generated from aarch64_ad.m4
13658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13659 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13660 %{
13661   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13662   ins_cost(INSN_COST);
13663   format %{ "add  $dst, $src1, $src2, sxtb" %}
13664 
13665    ins_encode %{
13666      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13667             as_Register($src2$$reg), ext::sxtb);
13668    %}
13669   ins_pipe(ialu_reg_reg);
13670 %}
13671 
13672 // This pattern is automatically generated from aarch64_ad.m4
13673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13674 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13675 %{
13676   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13677   ins_cost(INSN_COST);
13678   format %{ "add  $dst, $src1, $src2, uxtb" %}
13679 
13680    ins_encode %{
13681      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13682             as_Register($src2$$reg), ext::uxtb);
13683    %}
13684   ins_pipe(ialu_reg_reg);
13685 %}
13686 
13687 // This pattern is automatically generated from aarch64_ad.m4
13688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13689 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13690 %{
13691   match(Set dst (AddI src1 (AndI src2 mask)));
13692   ins_cost(INSN_COST);
13693   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13694 
13695    ins_encode %{
13696      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13697             as_Register($src2$$reg), ext::uxtb);
13698    %}
13699   ins_pipe(ialu_reg_reg);
13700 %}
13701 
13702 // This pattern is automatically generated from aarch64_ad.m4
13703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13704 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13705 %{
13706   match(Set dst (AddI src1 (AndI src2 mask)));
13707   ins_cost(INSN_COST);
13708   format %{ "addw  $dst, $src1, $src2, uxth" %}
13709 
13710    ins_encode %{
13711      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13712             as_Register($src2$$reg), ext::uxth);
13713    %}
13714   ins_pipe(ialu_reg_reg);
13715 %}
13716 
13717 // This pattern is automatically generated from aarch64_ad.m4
13718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13719 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13720 %{
13721   match(Set dst (AddL src1 (AndL src2 mask)));
13722   ins_cost(INSN_COST);
13723   format %{ "add  $dst, $src1, $src2, uxtb" %}
13724 
13725    ins_encode %{
13726      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13727             as_Register($src2$$reg), ext::uxtb);
13728    %}
13729   ins_pipe(ialu_reg_reg);
13730 %}
13731 
13732 // This pattern is automatically generated from aarch64_ad.m4
13733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13734 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13735 %{
13736   match(Set dst (AddL src1 (AndL src2 mask)));
13737   ins_cost(INSN_COST);
13738   format %{ "add  $dst, $src1, $src2, uxth" %}
13739 
13740    ins_encode %{
13741      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13742             as_Register($src2$$reg), ext::uxth);
13743    %}
13744   ins_pipe(ialu_reg_reg);
13745 %}
13746 
13747 // This pattern is automatically generated from aarch64_ad.m4
13748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13749 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13750 %{
13751   match(Set dst (AddL src1 (AndL src2 mask)));
13752   ins_cost(INSN_COST);
13753   format %{ "add  $dst, $src1, $src2, uxtw" %}
13754 
13755    ins_encode %{
13756      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13757             as_Register($src2$$reg), ext::uxtw);
13758    %}
13759   ins_pipe(ialu_reg_reg);
13760 %}
13761 
13762 // This pattern is automatically generated from aarch64_ad.m4
13763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13764 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13765 %{
13766   match(Set dst (SubI src1 (AndI src2 mask)));
13767   ins_cost(INSN_COST);
13768   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13769 
13770    ins_encode %{
13771      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13772             as_Register($src2$$reg), ext::uxtb);
13773    %}
13774   ins_pipe(ialu_reg_reg);
13775 %}
13776 
13777 // This pattern is automatically generated from aarch64_ad.m4
13778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13779 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13780 %{
13781   match(Set dst (SubI src1 (AndI src2 mask)));
13782   ins_cost(INSN_COST);
13783   format %{ "subw  $dst, $src1, $src2, uxth" %}
13784 
13785    ins_encode %{
13786      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13787             as_Register($src2$$reg), ext::uxth);
13788    %}
13789   ins_pipe(ialu_reg_reg);
13790 %}
13791 
13792 // This pattern is automatically generated from aarch64_ad.m4
13793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13794 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13795 %{
13796   match(Set dst (SubL src1 (AndL src2 mask)));
13797   ins_cost(INSN_COST);
13798   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13799 
13800    ins_encode %{
13801      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13802             as_Register($src2$$reg), ext::uxtb);
13803    %}
13804   ins_pipe(ialu_reg_reg);
13805 %}
13806 
13807 // This pattern is automatically generated from aarch64_ad.m4
13808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13809 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13810 %{
13811   match(Set dst (SubL src1 (AndL src2 mask)));
13812   ins_cost(INSN_COST);
13813   format %{ "sub  $dst, $src1, $src2, uxth" %}
13814 
13815    ins_encode %{
13816      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13817             as_Register($src2$$reg), ext::uxth);
13818    %}
13819   ins_pipe(ialu_reg_reg);
13820 %}
13821 
13822 // This pattern is automatically generated from aarch64_ad.m4
13823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13824 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13825 %{
13826   match(Set dst (SubL src1 (AndL src2 mask)));
13827   ins_cost(INSN_COST);
13828   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13829 
13830    ins_encode %{
13831      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13832             as_Register($src2$$reg), ext::uxtw);
13833    %}
13834   ins_pipe(ialu_reg_reg);
13835 %}
13836 
13837 
13838 // This pattern is automatically generated from aarch64_ad.m4
13839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13840 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13841 %{
13842   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13843   ins_cost(1.9 * INSN_COST);
13844   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13845 
13846    ins_encode %{
13847      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13848             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13849    %}
13850   ins_pipe(ialu_reg_reg_shift);
13851 %}
13852 
13853 // This pattern is automatically generated from aarch64_ad.m4
13854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13855 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13856 %{
13857   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13858   ins_cost(1.9 * INSN_COST);
13859   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13860 
13861    ins_encode %{
13862      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13863             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13864    %}
13865   ins_pipe(ialu_reg_reg_shift);
13866 %}
13867 
13868 // This pattern is automatically generated from aarch64_ad.m4
13869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13870 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13871 %{
13872   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13873   ins_cost(1.9 * INSN_COST);
13874   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13875 
13876    ins_encode %{
13877      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13878             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13879    %}
13880   ins_pipe(ialu_reg_reg_shift);
13881 %}
13882 
13883 // This pattern is automatically generated from aarch64_ad.m4
13884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13885 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13886 %{
13887   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13888   ins_cost(1.9 * INSN_COST);
13889   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13890 
13891    ins_encode %{
13892      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13893             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13894    %}
13895   ins_pipe(ialu_reg_reg_shift);
13896 %}
13897 
13898 // This pattern is automatically generated from aarch64_ad.m4
13899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13900 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13901 %{
13902   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13903   ins_cost(1.9 * INSN_COST);
13904   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13905 
13906    ins_encode %{
13907      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13908             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13909    %}
13910   ins_pipe(ialu_reg_reg_shift);
13911 %}
13912 
13913 // This pattern is automatically generated from aarch64_ad.m4
13914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13915 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13916 %{
13917   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13918   ins_cost(1.9 * INSN_COST);
13919   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13920 
13921    ins_encode %{
13922      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13923             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13924    %}
13925   ins_pipe(ialu_reg_reg_shift);
13926 %}
13927 
13928 // This pattern is automatically generated from aarch64_ad.m4
13929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13930 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13931 %{
13932   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13933   ins_cost(1.9 * INSN_COST);
13934   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13935 
13936    ins_encode %{
13937      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13938             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13939    %}
13940   ins_pipe(ialu_reg_reg_shift);
13941 %}
13942 
13943 // This pattern is automatically generated from aarch64_ad.m4
13944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13945 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13946 %{
13947   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13948   ins_cost(1.9 * INSN_COST);
13949   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13950 
13951    ins_encode %{
13952      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13953             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13954    %}
13955   ins_pipe(ialu_reg_reg_shift);
13956 %}
13957 
13958 // This pattern is automatically generated from aarch64_ad.m4
13959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13960 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13961 %{
13962   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13963   ins_cost(1.9 * INSN_COST);
13964   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13965 
13966    ins_encode %{
13967      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13968             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13969    %}
13970   ins_pipe(ialu_reg_reg_shift);
13971 %}
13972 
13973 // This pattern is automatically generated from aarch64_ad.m4
13974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13975 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13976 %{
13977   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13978   ins_cost(1.9 * INSN_COST);
13979   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13980 
13981    ins_encode %{
13982      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13983             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13984    %}
13985   ins_pipe(ialu_reg_reg_shift);
13986 %}
13987 
13988 // This pattern is automatically generated from aarch64_ad.m4
13989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13990 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13991 %{
13992   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13993   ins_cost(1.9 * INSN_COST);
13994   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13995 
13996    ins_encode %{
13997      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13998             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13999    %}
14000   ins_pipe(ialu_reg_reg_shift);
14001 %}
14002 
14003 // This pattern is automatically generated from aarch64_ad.m4
14004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14005 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
14006 %{
14007   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
14008   ins_cost(1.9 * INSN_COST);
14009   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
14010 
14011    ins_encode %{
14012      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14013             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
14014    %}
14015   ins_pipe(ialu_reg_reg_shift);
14016 %}
14017 
14018 // This pattern is automatically generated from aarch64_ad.m4
14019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14020 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14021 %{
14022   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14023   ins_cost(1.9 * INSN_COST);
14024   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
14025 
14026    ins_encode %{
14027      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14028             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14029    %}
14030   ins_pipe(ialu_reg_reg_shift);
14031 %}
14032 
14033 // This pattern is automatically generated from aarch64_ad.m4
14034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14035 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14036 %{
14037   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14038   ins_cost(1.9 * INSN_COST);
14039   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
14040 
14041    ins_encode %{
14042      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14043             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14044    %}
14045   ins_pipe(ialu_reg_reg_shift);
14046 %}
14047 
14048 // This pattern is automatically generated from aarch64_ad.m4
14049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14050 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14051 %{
14052   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14053   ins_cost(1.9 * INSN_COST);
14054   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
14055 
14056    ins_encode %{
14057      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14058             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14059    %}
14060   ins_pipe(ialu_reg_reg_shift);
14061 %}
14062 
14063 // This pattern is automatically generated from aarch64_ad.m4
14064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14065 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14066 %{
14067   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14068   ins_cost(1.9 * INSN_COST);
14069   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
14070 
14071    ins_encode %{
14072      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14073             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14074    %}
14075   ins_pipe(ialu_reg_reg_shift);
14076 %}
14077 
14078 // This pattern is automatically generated from aarch64_ad.m4
14079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14080 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14081 %{
14082   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14083   ins_cost(1.9 * INSN_COST);
14084   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
14085 
14086    ins_encode %{
14087      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14088             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14089    %}
14090   ins_pipe(ialu_reg_reg_shift);
14091 %}
14092 
14093 // This pattern is automatically generated from aarch64_ad.m4
14094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14095 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14096 %{
14097   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14098   ins_cost(1.9 * INSN_COST);
14099   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
14100 
14101    ins_encode %{
14102      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14103             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14104    %}
14105   ins_pipe(ialu_reg_reg_shift);
14106 %}
14107 
14108 // This pattern is automatically generated from aarch64_ad.m4
14109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14110 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14111 %{
14112   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14113   ins_cost(1.9 * INSN_COST);
14114   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
14115 
14116    ins_encode %{
14117      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14118             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14119    %}
14120   ins_pipe(ialu_reg_reg_shift);
14121 %}
14122 
14123 // This pattern is automatically generated from aarch64_ad.m4
14124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14125 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14126 %{
14127   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14128   ins_cost(1.9 * INSN_COST);
14129   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
14130 
14131    ins_encode %{
14132      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14133             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14134    %}
14135   ins_pipe(ialu_reg_reg_shift);
14136 %}
14137 
14138 // This pattern is automatically generated from aarch64_ad.m4
14139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14140 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14141 %{
14142   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14143   ins_cost(1.9 * INSN_COST);
14144   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
14145 
14146    ins_encode %{
14147      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14148             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14149    %}
14150   ins_pipe(ialu_reg_reg_shift);
14151 %}
14152 
14153 // This pattern is automatically generated from aarch64_ad.m4
14154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14155 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14156 %{
14157   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14158   ins_cost(1.9 * INSN_COST);
14159   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
14160 
14161    ins_encode %{
14162      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14163             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14164    %}
14165   ins_pipe(ialu_reg_reg_shift);
14166 %}
14167 
14168 
14169 
14170 // END This section of the file is automatically generated. Do not edit --------------
14171 
14172 
14173 // ============================================================================
14174 // Floating Point Arithmetic Instructions
14175 
14176 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14177   match(Set dst (AddF src1 src2));
14178 
14179   ins_cost(INSN_COST * 5);
14180   format %{ "fadds   $dst, $src1, $src2" %}
14181 
14182   ins_encode %{
14183     __ fadds(as_FloatRegister($dst$$reg),
14184              as_FloatRegister($src1$$reg),
14185              as_FloatRegister($src2$$reg));
14186   %}
14187 
14188   ins_pipe(fp_dop_reg_reg_s);
14189 %}
14190 
14191 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14192   match(Set dst (AddD src1 src2));
14193 
14194   ins_cost(INSN_COST * 5);
14195   format %{ "faddd   $dst, $src1, $src2" %}
14196 
14197   ins_encode %{
14198     __ faddd(as_FloatRegister($dst$$reg),
14199              as_FloatRegister($src1$$reg),
14200              as_FloatRegister($src2$$reg));
14201   %}
14202 
14203   ins_pipe(fp_dop_reg_reg_d);
14204 %}
14205 
14206 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14207   match(Set dst (SubF src1 src2));
14208 
14209   ins_cost(INSN_COST * 5);
14210   format %{ "fsubs   $dst, $src1, $src2" %}
14211 
14212   ins_encode %{
14213     __ fsubs(as_FloatRegister($dst$$reg),
14214              as_FloatRegister($src1$$reg),
14215              as_FloatRegister($src2$$reg));
14216   %}
14217 
14218   ins_pipe(fp_dop_reg_reg_s);
14219 %}
14220 
14221 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14222   match(Set dst (SubD src1 src2));
14223 
14224   ins_cost(INSN_COST * 5);
14225   format %{ "fsubd   $dst, $src1, $src2" %}
14226 
14227   ins_encode %{
14228     __ fsubd(as_FloatRegister($dst$$reg),
14229              as_FloatRegister($src1$$reg),
14230              as_FloatRegister($src2$$reg));
14231   %}
14232 
14233   ins_pipe(fp_dop_reg_reg_d);
14234 %}
14235 
14236 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14237   match(Set dst (MulF src1 src2));
14238 
14239   ins_cost(INSN_COST * 6);
14240   format %{ "fmuls   $dst, $src1, $src2" %}
14241 
14242   ins_encode %{
14243     __ fmuls(as_FloatRegister($dst$$reg),
14244              as_FloatRegister($src1$$reg),
14245              as_FloatRegister($src2$$reg));
14246   %}
14247 
14248   ins_pipe(fp_dop_reg_reg_s);
14249 %}
14250 
14251 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14252   match(Set dst (MulD src1 src2));
14253 
14254   ins_cost(INSN_COST * 6);
14255   format %{ "fmuld   $dst, $src1, $src2" %}
14256 
14257   ins_encode %{
14258     __ fmuld(as_FloatRegister($dst$$reg),
14259              as_FloatRegister($src1$$reg),
14260              as_FloatRegister($src2$$reg));
14261   %}
14262 
14263   ins_pipe(fp_dop_reg_reg_d);
14264 %}
14265 
14266 // src1 * src2 + src3
14267 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14268   predicate(UseFMA);
14269   match(Set dst (FmaF src3 (Binary src1 src2)));
14270 
14271   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14272 
14273   ins_encode %{
14274     __ fmadds(as_FloatRegister($dst$$reg),
14275              as_FloatRegister($src1$$reg),
14276              as_FloatRegister($src2$$reg),
14277              as_FloatRegister($src3$$reg));
14278   %}
14279 
14280   ins_pipe(pipe_class_default);
14281 %}
14282 
14283 // src1 * src2 + src3
14284 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14285   predicate(UseFMA);
14286   match(Set dst (FmaD src3 (Binary src1 src2)));
14287 
14288   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14289 
14290   ins_encode %{
14291     __ fmaddd(as_FloatRegister($dst$$reg),
14292              as_FloatRegister($src1$$reg),
14293              as_FloatRegister($src2$$reg),
14294              as_FloatRegister($src3$$reg));
14295   %}
14296 
14297   ins_pipe(pipe_class_default);
14298 %}
14299 
14300 // -src1 * src2 + src3
14301 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14302   predicate(UseFMA);
14303   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14304   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14305 
14306   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14307 
14308   ins_encode %{
14309     __ fmsubs(as_FloatRegister($dst$$reg),
14310               as_FloatRegister($src1$$reg),
14311               as_FloatRegister($src2$$reg),
14312               as_FloatRegister($src3$$reg));
14313   %}
14314 
14315   ins_pipe(pipe_class_default);
14316 %}
14317 
14318 // -src1 * src2 + src3
14319 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14320   predicate(UseFMA);
14321   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14322   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14323 
14324   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14325 
14326   ins_encode %{
14327     __ fmsubd(as_FloatRegister($dst$$reg),
14328               as_FloatRegister($src1$$reg),
14329               as_FloatRegister($src2$$reg),
14330               as_FloatRegister($src3$$reg));
14331   %}
14332 
14333   ins_pipe(pipe_class_default);
14334 %}
14335 
14336 // -src1 * src2 - src3
14337 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14338   predicate(UseFMA);
14339   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14340   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14341 
14342   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14343 
14344   ins_encode %{
14345     __ fnmadds(as_FloatRegister($dst$$reg),
14346                as_FloatRegister($src1$$reg),
14347                as_FloatRegister($src2$$reg),
14348                as_FloatRegister($src3$$reg));
14349   %}
14350 
14351   ins_pipe(pipe_class_default);
14352 %}
14353 
14354 // -src1 * src2 - src3
14355 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14356   predicate(UseFMA);
14357   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14358   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14359 
14360   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14361 
14362   ins_encode %{
14363     __ fnmaddd(as_FloatRegister($dst$$reg),
14364                as_FloatRegister($src1$$reg),
14365                as_FloatRegister($src2$$reg),
14366                as_FloatRegister($src3$$reg));
14367   %}
14368 
14369   ins_pipe(pipe_class_default);
14370 %}
14371 
14372 // src1 * src2 - src3
14373 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14374   predicate(UseFMA);
14375   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14376 
14377   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14378 
14379   ins_encode %{
14380     __ fnmsubs(as_FloatRegister($dst$$reg),
14381                as_FloatRegister($src1$$reg),
14382                as_FloatRegister($src2$$reg),
14383                as_FloatRegister($src3$$reg));
14384   %}
14385 
14386   ins_pipe(pipe_class_default);
14387 %}
14388 
14389 // src1 * src2 - src3
14390 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14391   predicate(UseFMA);
14392   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14393 
14394   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14395 
14396   ins_encode %{
14397   // n.b. insn name should be fnmsubd
14398     __ fnmsub(as_FloatRegister($dst$$reg),
14399               as_FloatRegister($src1$$reg),
14400               as_FloatRegister($src2$$reg),
14401               as_FloatRegister($src3$$reg));
14402   %}
14403 
14404   ins_pipe(pipe_class_default);
14405 %}
14406 
14407 
14408 // Math.max(FF)F
14409 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14410   match(Set dst (MaxF src1 src2));
14411 
14412   format %{ "fmaxs   $dst, $src1, $src2" %}
14413   ins_encode %{
14414     __ fmaxs(as_FloatRegister($dst$$reg),
14415              as_FloatRegister($src1$$reg),
14416              as_FloatRegister($src2$$reg));
14417   %}
14418 
14419   ins_pipe(fp_dop_reg_reg_s);
14420 %}
14421 
14422 // Math.min(FF)F
14423 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14424   match(Set dst (MinF src1 src2));
14425 
14426   format %{ "fmins   $dst, $src1, $src2" %}
14427   ins_encode %{
14428     __ fmins(as_FloatRegister($dst$$reg),
14429              as_FloatRegister($src1$$reg),
14430              as_FloatRegister($src2$$reg));
14431   %}
14432 
14433   ins_pipe(fp_dop_reg_reg_s);
14434 %}
14435 
14436 // Math.max(DD)D
14437 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14438   match(Set dst (MaxD src1 src2));
14439 
14440   format %{ "fmaxd   $dst, $src1, $src2" %}
14441   ins_encode %{
14442     __ fmaxd(as_FloatRegister($dst$$reg),
14443              as_FloatRegister($src1$$reg),
14444              as_FloatRegister($src2$$reg));
14445   %}
14446 
14447   ins_pipe(fp_dop_reg_reg_d);
14448 %}
14449 
14450 // Math.min(DD)D
14451 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14452   match(Set dst (MinD src1 src2));
14453 
14454   format %{ "fmind   $dst, $src1, $src2" %}
14455   ins_encode %{
14456     __ fmind(as_FloatRegister($dst$$reg),
14457              as_FloatRegister($src1$$reg),
14458              as_FloatRegister($src2$$reg));
14459   %}
14460 
14461   ins_pipe(fp_dop_reg_reg_d);
14462 %}
14463 
14464 
14465 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14466   match(Set dst (DivF src1  src2));
14467 
14468   ins_cost(INSN_COST * 18);
14469   format %{ "fdivs   $dst, $src1, $src2" %}
14470 
14471   ins_encode %{
14472     __ fdivs(as_FloatRegister($dst$$reg),
14473              as_FloatRegister($src1$$reg),
14474              as_FloatRegister($src2$$reg));
14475   %}
14476 
14477   ins_pipe(fp_div_s);
14478 %}
14479 
14480 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14481   match(Set dst (DivD src1  src2));
14482 
14483   ins_cost(INSN_COST * 32);
14484   format %{ "fdivd   $dst, $src1, $src2" %}
14485 
14486   ins_encode %{
14487     __ fdivd(as_FloatRegister($dst$$reg),
14488              as_FloatRegister($src1$$reg),
14489              as_FloatRegister($src2$$reg));
14490   %}
14491 
14492   ins_pipe(fp_div_d);
14493 %}
14494 
14495 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14496   match(Set dst (NegF src));
14497 
14498   ins_cost(INSN_COST * 3);
14499   format %{ "fneg   $dst, $src" %}
14500 
14501   ins_encode %{
14502     __ fnegs(as_FloatRegister($dst$$reg),
14503              as_FloatRegister($src$$reg));
14504   %}
14505 
14506   ins_pipe(fp_uop_s);
14507 %}
14508 
14509 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14510   match(Set dst (NegD src));
14511 
14512   ins_cost(INSN_COST * 3);
14513   format %{ "fnegd   $dst, $src" %}
14514 
14515   ins_encode %{
14516     __ fnegd(as_FloatRegister($dst$$reg),
14517              as_FloatRegister($src$$reg));
14518   %}
14519 
14520   ins_pipe(fp_uop_d);
14521 %}
14522 
14523 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14524 %{
14525   match(Set dst (AbsI src));
14526 
14527   effect(KILL cr);
14528   ins_cost(INSN_COST * 2);
14529   format %{ "cmpw  $src, zr\n\t"
14530             "cnegw $dst, $src, Assembler::LT\t# int abs"
14531   %}
14532 
14533   ins_encode %{
14534     __ cmpw(as_Register($src$$reg), zr);
14535     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14536   %}
14537   ins_pipe(pipe_class_default);
14538 %}
14539 
14540 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14541 %{
14542   match(Set dst (AbsL src));
14543 
14544   effect(KILL cr);
14545   ins_cost(INSN_COST * 2);
14546   format %{ "cmp  $src, zr\n\t"
14547             "cneg $dst, $src, Assembler::LT\t# long abs"
14548   %}
14549 
14550   ins_encode %{
14551     __ cmp(as_Register($src$$reg), zr);
14552     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14553   %}
14554   ins_pipe(pipe_class_default);
14555 %}
14556 
14557 instruct absF_reg(vRegF dst, vRegF src) %{
14558   match(Set dst (AbsF src));
14559 
14560   ins_cost(INSN_COST * 3);
14561   format %{ "fabss   $dst, $src" %}
14562   ins_encode %{
14563     __ fabss(as_FloatRegister($dst$$reg),
14564              as_FloatRegister($src$$reg));
14565   %}
14566 
14567   ins_pipe(fp_uop_s);
14568 %}
14569 
14570 instruct absD_reg(vRegD dst, vRegD src) %{
14571   match(Set dst (AbsD src));
14572 
14573   ins_cost(INSN_COST * 3);
14574   format %{ "fabsd   $dst, $src" %}
14575   ins_encode %{
14576     __ fabsd(as_FloatRegister($dst$$reg),
14577              as_FloatRegister($src$$reg));
14578   %}
14579 
14580   ins_pipe(fp_uop_d);
14581 %}
14582 
14583 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14584   match(Set dst (AbsF (SubF src1 src2)));
14585 
14586   ins_cost(INSN_COST * 3);
14587   format %{ "fabds   $dst, $src1, $src2" %}
14588   ins_encode %{
14589     __ fabds(as_FloatRegister($dst$$reg),
14590              as_FloatRegister($src1$$reg),
14591              as_FloatRegister($src2$$reg));
14592   %}
14593 
14594   ins_pipe(fp_uop_s);
14595 %}
14596 
14597 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14598   match(Set dst (AbsD (SubD src1 src2)));
14599 
14600   ins_cost(INSN_COST * 3);
14601   format %{ "fabdd   $dst, $src1, $src2" %}
14602   ins_encode %{
14603     __ fabdd(as_FloatRegister($dst$$reg),
14604              as_FloatRegister($src1$$reg),
14605              as_FloatRegister($src2$$reg));
14606   %}
14607 
14608   ins_pipe(fp_uop_d);
14609 %}
14610 
14611 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14612   match(Set dst (SqrtD src));
14613 
14614   ins_cost(INSN_COST * 50);
14615   format %{ "fsqrtd  $dst, $src" %}
14616   ins_encode %{
14617     __ fsqrtd(as_FloatRegister($dst$$reg),
14618              as_FloatRegister($src$$reg));
14619   %}
14620 
14621   ins_pipe(fp_div_s);
14622 %}
14623 
14624 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14625   match(Set dst (SqrtF src));
14626 
14627   ins_cost(INSN_COST * 50);
14628   format %{ "fsqrts  $dst, $src" %}
14629   ins_encode %{
14630     __ fsqrts(as_FloatRegister($dst$$reg),
14631              as_FloatRegister($src$$reg));
14632   %}
14633 
14634   ins_pipe(fp_div_d);
14635 %}
14636 
14637 // Math.rint, floor, ceil
14638 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14639   match(Set dst (RoundDoubleMode src rmode));
14640   format %{ "frint  $dst, $src, $rmode" %}
14641   ins_encode %{
14642     switch ($rmode$$constant) {
14643       case RoundDoubleModeNode::rmode_rint:
14644         __ frintnd(as_FloatRegister($dst$$reg),
14645                    as_FloatRegister($src$$reg));
14646         break;
14647       case RoundDoubleModeNode::rmode_floor:
14648         __ frintmd(as_FloatRegister($dst$$reg),
14649                    as_FloatRegister($src$$reg));
14650         break;
14651       case RoundDoubleModeNode::rmode_ceil:
14652         __ frintpd(as_FloatRegister($dst$$reg),
14653                    as_FloatRegister($src$$reg));
14654         break;
14655     }
14656   %}
14657   ins_pipe(fp_uop_d);
14658 %}
14659 
14660 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14661   match(Set dst (CopySignD src1 (Binary src2 zero)));
14662   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14663   format %{ "CopySignD  $dst $src1 $src2" %}
14664   ins_encode %{
14665     FloatRegister dst = as_FloatRegister($dst$$reg),
14666                   src1 = as_FloatRegister($src1$$reg),
14667                   src2 = as_FloatRegister($src2$$reg),
14668                   zero = as_FloatRegister($zero$$reg);
14669     __ fnegd(dst, zero);
14670     __ bsl(dst, __ T8B, src2, src1);
14671   %}
14672   ins_pipe(fp_uop_d);
14673 %}
14674 
14675 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14676   match(Set dst (CopySignF src1 src2));
14677   effect(TEMP_DEF dst, USE src1, USE src2);
14678   format %{ "CopySignF  $dst $src1 $src2" %}
14679   ins_encode %{
14680     FloatRegister dst = as_FloatRegister($dst$$reg),
14681                   src1 = as_FloatRegister($src1$$reg),
14682                   src2 = as_FloatRegister($src2$$reg);
14683     __ movi(dst, __ T2S, 0x80, 24);
14684     __ bsl(dst, __ T8B, src2, src1);
14685   %}
14686   ins_pipe(fp_uop_d);
14687 %}
14688 
14689 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14690   match(Set dst (SignumD src (Binary zero one)));
14691   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14692   format %{ "signumD  $dst, $src" %}
14693   ins_encode %{
14694     FloatRegister src = as_FloatRegister($src$$reg),
14695                   dst = as_FloatRegister($dst$$reg),
14696                   zero = as_FloatRegister($zero$$reg),
14697                   one = as_FloatRegister($one$$reg);
14698     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14699     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14700     // Bit selection instruction gets bit from "one" for each enabled bit in
14701     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14702     // NaN the whole "src" will be copied because "dst" is zero. For all other
14703     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14704     // from "src", and all other bits are copied from 1.0.
14705     __ bsl(dst, __ T8B, one, src);
14706   %}
14707   ins_pipe(fp_uop_d);
14708 %}
14709 
14710 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14711   match(Set dst (SignumF src (Binary zero one)));
14712   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14713   format %{ "signumF  $dst, $src" %}
14714   ins_encode %{
14715     FloatRegister src = as_FloatRegister($src$$reg),
14716                   dst = as_FloatRegister($dst$$reg),
14717                   zero = as_FloatRegister($zero$$reg),
14718                   one = as_FloatRegister($one$$reg);
14719     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14720     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14721     // Bit selection instruction gets bit from "one" for each enabled bit in
14722     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14723     // NaN the whole "src" will be copied because "dst" is zero. For all other
14724     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14725     // from "src", and all other bits are copied from 1.0.
14726     __ bsl(dst, __ T8B, one, src);
14727   %}
14728   ins_pipe(fp_uop_d);
14729 %}
14730 
14731 instruct onspinwait() %{
14732   match(OnSpinWait);
14733   ins_cost(INSN_COST);
14734 
14735   format %{ "onspinwait" %}
14736 
14737   ins_encode %{
14738     __ spin_wait();
14739   %}
14740   ins_pipe(pipe_class_empty);
14741 %}
14742 
14743 // ============================================================================
14744 // Logical Instructions
14745 
14746 // Integer Logical Instructions
14747 
14748 // And Instructions
14749 
14750 
14751 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14752   match(Set dst (AndI src1 src2));
14753 
14754   format %{ "andw  $dst, $src1, $src2\t# int" %}
14755 
14756   ins_cost(INSN_COST);
14757   ins_encode %{
14758     __ andw(as_Register($dst$$reg),
14759             as_Register($src1$$reg),
14760             as_Register($src2$$reg));
14761   %}
14762 
14763   ins_pipe(ialu_reg_reg);
14764 %}
14765 
14766 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14767   match(Set dst (AndI src1 src2));
14768 
14769   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14770 
14771   ins_cost(INSN_COST);
14772   ins_encode %{
14773     __ andw(as_Register($dst$$reg),
14774             as_Register($src1$$reg),
14775             (uint64_t)($src2$$constant));
14776   %}
14777 
14778   ins_pipe(ialu_reg_imm);
14779 %}
14780 
14781 // Or Instructions
14782 
14783 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14784   match(Set dst (OrI src1 src2));
14785 
14786   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14787 
14788   ins_cost(INSN_COST);
14789   ins_encode %{
14790     __ orrw(as_Register($dst$$reg),
14791             as_Register($src1$$reg),
14792             as_Register($src2$$reg));
14793   %}
14794 
14795   ins_pipe(ialu_reg_reg);
14796 %}
14797 
14798 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14799   match(Set dst (OrI src1 src2));
14800 
14801   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14802 
14803   ins_cost(INSN_COST);
14804   ins_encode %{
14805     __ orrw(as_Register($dst$$reg),
14806             as_Register($src1$$reg),
14807             (uint64_t)($src2$$constant));
14808   %}
14809 
14810   ins_pipe(ialu_reg_imm);
14811 %}
14812 
14813 // Xor Instructions
14814 
14815 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14816   match(Set dst (XorI src1 src2));
14817 
14818   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14819 
14820   ins_cost(INSN_COST);
14821   ins_encode %{
14822     __ eorw(as_Register($dst$$reg),
14823             as_Register($src1$$reg),
14824             as_Register($src2$$reg));
14825   %}
14826 
14827   ins_pipe(ialu_reg_reg);
14828 %}
14829 
14830 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14831   match(Set dst (XorI src1 src2));
14832 
14833   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14834 
14835   ins_cost(INSN_COST);
14836   ins_encode %{
14837     __ eorw(as_Register($dst$$reg),
14838             as_Register($src1$$reg),
14839             (uint64_t)($src2$$constant));
14840   %}
14841 
14842   ins_pipe(ialu_reg_imm);
14843 %}
14844 
14845 // Long Logical Instructions
14846 // TODO
14847 
14848 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14849   match(Set dst (AndL src1 src2));
14850 
14851   format %{ "and  $dst, $src1, $src2\t# int" %}
14852 
14853   ins_cost(INSN_COST);
14854   ins_encode %{
14855     __ andr(as_Register($dst$$reg),
14856             as_Register($src1$$reg),
14857             as_Register($src2$$reg));
14858   %}
14859 
14860   ins_pipe(ialu_reg_reg);
14861 %}
14862 
14863 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14864   match(Set dst (AndL src1 src2));
14865 
14866   format %{ "and  $dst, $src1, $src2\t# int" %}
14867 
14868   ins_cost(INSN_COST);
14869   ins_encode %{
14870     __ andr(as_Register($dst$$reg),
14871             as_Register($src1$$reg),
14872             (uint64_t)($src2$$constant));
14873   %}
14874 
14875   ins_pipe(ialu_reg_imm);
14876 %}
14877 
14878 // Or Instructions
14879 
14880 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14881   match(Set dst (OrL src1 src2));
14882 
14883   format %{ "orr  $dst, $src1, $src2\t# int" %}
14884 
14885   ins_cost(INSN_COST);
14886   ins_encode %{
14887     __ orr(as_Register($dst$$reg),
14888            as_Register($src1$$reg),
14889            as_Register($src2$$reg));
14890   %}
14891 
14892   ins_pipe(ialu_reg_reg);
14893 %}
14894 
14895 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14896   match(Set dst (OrL src1 src2));
14897 
14898   format %{ "orr  $dst, $src1, $src2\t# int" %}
14899 
14900   ins_cost(INSN_COST);
14901   ins_encode %{
14902     __ orr(as_Register($dst$$reg),
14903            as_Register($src1$$reg),
14904            (uint64_t)($src2$$constant));
14905   %}
14906 
14907   ins_pipe(ialu_reg_imm);
14908 %}
14909 
14910 // Xor Instructions
14911 
14912 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14913   match(Set dst (XorL src1 src2));
14914 
14915   format %{ "eor  $dst, $src1, $src2\t# int" %}
14916 
14917   ins_cost(INSN_COST);
14918   ins_encode %{
14919     __ eor(as_Register($dst$$reg),
14920            as_Register($src1$$reg),
14921            as_Register($src2$$reg));
14922   %}
14923 
14924   ins_pipe(ialu_reg_reg);
14925 %}
14926 
14927 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14928   match(Set dst (XorL src1 src2));
14929 
14930   ins_cost(INSN_COST);
14931   format %{ "eor  $dst, $src1, $src2\t# int" %}
14932 
14933   ins_encode %{
14934     __ eor(as_Register($dst$$reg),
14935            as_Register($src1$$reg),
14936            (uint64_t)($src2$$constant));
14937   %}
14938 
14939   ins_pipe(ialu_reg_imm);
14940 %}
14941 
14942 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14943 %{
14944   match(Set dst (ConvI2L src));
14945 
14946   ins_cost(INSN_COST);
14947   format %{ "sxtw  $dst, $src\t# i2l" %}
14948   ins_encode %{
14949     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14950   %}
14951   ins_pipe(ialu_reg_shift);
14952 %}
14953 
14954 // this pattern occurs in bigmath arithmetic
14955 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14956 %{
14957   match(Set dst (AndL (ConvI2L src) mask));
14958 
14959   ins_cost(INSN_COST);
14960   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14961   ins_encode %{
14962     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14963   %}
14964 
14965   ins_pipe(ialu_reg_shift);
14966 %}
14967 
14968 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14969   match(Set dst (ConvL2I src));
14970 
14971   ins_cost(INSN_COST);
14972   format %{ "movw  $dst, $src \t// l2i" %}
14973 
14974   ins_encode %{
14975     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14976   %}
14977 
14978   ins_pipe(ialu_reg);
14979 %}
14980 
14981 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14982 %{
14983   match(Set dst (Conv2B src));
14984   effect(KILL cr);
14985 
14986   format %{
14987     "cmpw $src, zr\n\t"
14988     "cset $dst, ne"
14989   %}
14990 
14991   ins_encode %{
14992     __ cmpw(as_Register($src$$reg), zr);
14993     __ cset(as_Register($dst$$reg), Assembler::NE);
14994   %}
14995 
14996   ins_pipe(ialu_reg);
14997 %}
14998 
14999 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
15000 %{
15001   match(Set dst (Conv2B src));
15002   effect(KILL cr);
15003 
15004   format %{
15005     "cmp  $src, zr\n\t"
15006     "cset $dst, ne"
15007   %}
15008 
15009   ins_encode %{
15010     __ cmp(as_Register($src$$reg), zr);
15011     __ cset(as_Register($dst$$reg), Assembler::NE);
15012   %}
15013 
15014   ins_pipe(ialu_reg);
15015 %}
15016 
15017 instruct convD2F_reg(vRegF dst, vRegD src) %{
15018   match(Set dst (ConvD2F src));
15019 
15020   ins_cost(INSN_COST * 5);
15021   format %{ "fcvtd  $dst, $src \t// d2f" %}
15022 
15023   ins_encode %{
15024     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15025   %}
15026 
15027   ins_pipe(fp_d2f);
15028 %}
15029 
15030 instruct convF2D_reg(vRegD dst, vRegF src) %{
15031   match(Set dst (ConvF2D src));
15032 
15033   ins_cost(INSN_COST * 5);
15034   format %{ "fcvts  $dst, $src \t// f2d" %}
15035 
15036   ins_encode %{
15037     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15038   %}
15039 
15040   ins_pipe(fp_f2d);
15041 %}
15042 
15043 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15044   match(Set dst (ConvF2I src));
15045 
15046   ins_cost(INSN_COST * 5);
15047   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
15048 
15049   ins_encode %{
15050     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15051   %}
15052 
15053   ins_pipe(fp_f2i);
15054 %}
15055 
15056 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15057   match(Set dst (ConvF2L src));
15058 
15059   ins_cost(INSN_COST * 5);
15060   format %{ "fcvtzs  $dst, $src \t// f2l" %}
15061 
15062   ins_encode %{
15063     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15064   %}
15065 
15066   ins_pipe(fp_f2l);
15067 %}
15068 
15069 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15070   match(Set dst (ConvI2F src));
15071 
15072   ins_cost(INSN_COST * 5);
15073   format %{ "scvtfws  $dst, $src \t// i2f" %}
15074 
15075   ins_encode %{
15076     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15077   %}
15078 
15079   ins_pipe(fp_i2f);
15080 %}
15081 
15082 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15083   match(Set dst (ConvL2F src));
15084 
15085   ins_cost(INSN_COST * 5);
15086   format %{ "scvtfs  $dst, $src \t// l2f" %}
15087 
15088   ins_encode %{
15089     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15090   %}
15091 
15092   ins_pipe(fp_l2f);
15093 %}
15094 
15095 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15096   match(Set dst (ConvD2I src));
15097 
15098   ins_cost(INSN_COST * 5);
15099   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15100 
15101   ins_encode %{
15102     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15103   %}
15104 
15105   ins_pipe(fp_d2i);
15106 %}
15107 
15108 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15109   match(Set dst (ConvD2L src));
15110 
15111   ins_cost(INSN_COST * 5);
15112   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15113 
15114   ins_encode %{
15115     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15116   %}
15117 
15118   ins_pipe(fp_d2l);
15119 %}
15120 
15121 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15122   match(Set dst (ConvI2D src));
15123 
15124   ins_cost(INSN_COST * 5);
15125   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15126 
15127   ins_encode %{
15128     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15129   %}
15130 
15131   ins_pipe(fp_i2d);
15132 %}
15133 
15134 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15135   match(Set dst (ConvL2D src));
15136 
15137   ins_cost(INSN_COST * 5);
15138   format %{ "scvtfd  $dst, $src \t// l2d" %}
15139 
15140   ins_encode %{
15141     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15142   %}
15143 
15144   ins_pipe(fp_l2d);
15145 %}
15146 
15147 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15148 %{
15149   match(Set dst (RoundD src));
15150   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15151   format %{ "java_round_double $dst,$src"%}
15152   ins_encode %{
15153     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15154                          as_FloatRegister($ftmp$$reg));
15155   %}
15156   ins_pipe(pipe_slow);
15157 %}
15158 
15159 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15160 %{
15161   match(Set dst (RoundF src));
15162   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15163   format %{ "java_round_float $dst,$src"%}
15164   ins_encode %{
15165     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15166                         as_FloatRegister($ftmp$$reg));
15167   %}
15168   ins_pipe(pipe_slow);
15169 %}
15170 
15171 // stack <-> reg and reg <-> reg shuffles with no conversion
15172 
15173 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15174 
15175   match(Set dst (MoveF2I src));
15176 
15177   effect(DEF dst, USE src);
15178 
15179   ins_cost(4 * INSN_COST);
15180 
15181   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15182 
15183   ins_encode %{
15184     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15185   %}
15186 
15187   ins_pipe(iload_reg_reg);
15188 
15189 %}
15190 
15191 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15192 
15193   match(Set dst (MoveI2F src));
15194 
15195   effect(DEF dst, USE src);
15196 
15197   ins_cost(4 * INSN_COST);
15198 
15199   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15200 
15201   ins_encode %{
15202     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15203   %}
15204 
15205   ins_pipe(pipe_class_memory);
15206 
15207 %}
15208 
15209 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15210 
15211   match(Set dst (MoveD2L src));
15212 
15213   effect(DEF dst, USE src);
15214 
15215   ins_cost(4 * INSN_COST);
15216 
15217   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15218 
15219   ins_encode %{
15220     __ ldr($dst$$Register, Address(sp, $src$$disp));
15221   %}
15222 
15223   ins_pipe(iload_reg_reg);
15224 
15225 %}
15226 
15227 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15228 
15229   match(Set dst (MoveL2D src));
15230 
15231   effect(DEF dst, USE src);
15232 
15233   ins_cost(4 * INSN_COST);
15234 
15235   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15236 
15237   ins_encode %{
15238     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15239   %}
15240 
15241   ins_pipe(pipe_class_memory);
15242 
15243 %}
15244 
15245 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15246 
15247   match(Set dst (MoveF2I src));
15248 
15249   effect(DEF dst, USE src);
15250 
15251   ins_cost(INSN_COST);
15252 
15253   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15254 
15255   ins_encode %{
15256     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15257   %}
15258 
15259   ins_pipe(pipe_class_memory);
15260 
15261 %}
15262 
15263 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15264 
15265   match(Set dst (MoveI2F src));
15266 
15267   effect(DEF dst, USE src);
15268 
15269   ins_cost(INSN_COST);
15270 
15271   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15272 
15273   ins_encode %{
15274     __ strw($src$$Register, Address(sp, $dst$$disp));
15275   %}
15276 
15277   ins_pipe(istore_reg_reg);
15278 
15279 %}
15280 
15281 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15282 
15283   match(Set dst (MoveD2L src));
15284 
15285   effect(DEF dst, USE src);
15286 
15287   ins_cost(INSN_COST);
15288 
15289   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15290 
15291   ins_encode %{
15292     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15293   %}
15294 
15295   ins_pipe(pipe_class_memory);
15296 
15297 %}
15298 
15299 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15300 
15301   match(Set dst (MoveL2D src));
15302 
15303   effect(DEF dst, USE src);
15304 
15305   ins_cost(INSN_COST);
15306 
15307   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15308 
15309   ins_encode %{
15310     __ str($src$$Register, Address(sp, $dst$$disp));
15311   %}
15312 
15313   ins_pipe(istore_reg_reg);
15314 
15315 %}
15316 
15317 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15318 
15319   match(Set dst (MoveF2I src));
15320 
15321   effect(DEF dst, USE src);
15322 
15323   ins_cost(INSN_COST);
15324 
15325   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15326 
15327   ins_encode %{
15328     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15329   %}
15330 
15331   ins_pipe(fp_f2i);
15332 
15333 %}
15334 
15335 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15336 
15337   match(Set dst (MoveI2F src));
15338 
15339   effect(DEF dst, USE src);
15340 
15341   ins_cost(INSN_COST);
15342 
15343   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15344 
15345   ins_encode %{
15346     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15347   %}
15348 
15349   ins_pipe(fp_i2f);
15350 
15351 %}
15352 
15353 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15354 
15355   match(Set dst (MoveD2L src));
15356 
15357   effect(DEF dst, USE src);
15358 
15359   ins_cost(INSN_COST);
15360 
15361   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15362 
15363   ins_encode %{
15364     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15365   %}
15366 
15367   ins_pipe(fp_d2l);
15368 
15369 %}
15370 
15371 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15372 
15373   match(Set dst (MoveL2D src));
15374 
15375   effect(DEF dst, USE src);
15376 
15377   ins_cost(INSN_COST);
15378 
15379   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15380 
15381   ins_encode %{
15382     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15383   %}
15384 
15385   ins_pipe(fp_l2d);
15386 
15387 %}
15388 
15389 // ============================================================================
15390 // clearing of an array
15391 
15392 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15393 %{
15394   match(Set dummy (ClearArray cnt base));
15395   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15396 
15397   ins_cost(4 * INSN_COST);
15398   format %{ "ClearArray $cnt, $base" %}
15399 
15400   ins_encode %{
15401     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15402     if (tpc == NULL) {
15403       ciEnv::current()->record_failure("CodeCache is full");
15404       return;
15405     }
15406   %}
15407 
15408   ins_pipe(pipe_class_memory);
15409 %}
15410 
15411 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15412 %{
15413   predicate((uint64_t)n->in(2)->get_long()
15414             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15415   match(Set dummy (ClearArray cnt base));
15416   effect(TEMP temp, USE_KILL base, KILL cr);
15417 
15418   ins_cost(4 * INSN_COST);
15419   format %{ "ClearArray $cnt, $base" %}
15420 
15421   ins_encode %{
15422     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15423   %}
15424 
15425   ins_pipe(pipe_class_memory);
15426 %}
15427 
15428 // ============================================================================
15429 // Overflow Math Instructions
15430 
15431 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15432 %{
15433   match(Set cr (OverflowAddI op1 op2));
15434 
15435   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15436   ins_cost(INSN_COST);
15437   ins_encode %{
15438     __ cmnw($op1$$Register, $op2$$Register);
15439   %}
15440 
15441   ins_pipe(icmp_reg_reg);
15442 %}
15443 
15444 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15445 %{
15446   match(Set cr (OverflowAddI op1 op2));
15447 
15448   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15449   ins_cost(INSN_COST);
15450   ins_encode %{
15451     __ cmnw($op1$$Register, $op2$$constant);
15452   %}
15453 
15454   ins_pipe(icmp_reg_imm);
15455 %}
15456 
15457 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15458 %{
15459   match(Set cr (OverflowAddL op1 op2));
15460 
15461   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15462   ins_cost(INSN_COST);
15463   ins_encode %{
15464     __ cmn($op1$$Register, $op2$$Register);
15465   %}
15466 
15467   ins_pipe(icmp_reg_reg);
15468 %}
15469 
15470 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15471 %{
15472   match(Set cr (OverflowAddL op1 op2));
15473 
15474   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15475   ins_cost(INSN_COST);
15476   ins_encode %{
15477     __ cmn($op1$$Register, $op2$$constant);
15478   %}
15479 
15480   ins_pipe(icmp_reg_imm);
15481 %}
15482 
15483 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15484 %{
15485   match(Set cr (OverflowSubI op1 op2));
15486 
15487   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15488   ins_cost(INSN_COST);
15489   ins_encode %{
15490     __ cmpw($op1$$Register, $op2$$Register);
15491   %}
15492 
15493   ins_pipe(icmp_reg_reg);
15494 %}
15495 
15496 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15497 %{
15498   match(Set cr (OverflowSubI op1 op2));
15499 
15500   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15501   ins_cost(INSN_COST);
15502   ins_encode %{
15503     __ cmpw($op1$$Register, $op2$$constant);
15504   %}
15505 
15506   ins_pipe(icmp_reg_imm);
15507 %}
15508 
15509 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15510 %{
15511   match(Set cr (OverflowSubL op1 op2));
15512 
15513   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15514   ins_cost(INSN_COST);
15515   ins_encode %{
15516     __ cmp($op1$$Register, $op2$$Register);
15517   %}
15518 
15519   ins_pipe(icmp_reg_reg);
15520 %}
15521 
15522 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15523 %{
15524   match(Set cr (OverflowSubL op1 op2));
15525 
15526   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15527   ins_cost(INSN_COST);
15528   ins_encode %{
15529     __ subs(zr, $op1$$Register, $op2$$constant);
15530   %}
15531 
15532   ins_pipe(icmp_reg_imm);
15533 %}
15534 
15535 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15536 %{
15537   match(Set cr (OverflowSubI zero op1));
15538 
15539   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15540   ins_cost(INSN_COST);
15541   ins_encode %{
15542     __ cmpw(zr, $op1$$Register);
15543   %}
15544 
15545   ins_pipe(icmp_reg_imm);
15546 %}
15547 
15548 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15549 %{
15550   match(Set cr (OverflowSubL zero op1));
15551 
15552   format %{ "cmp   zr, $op1\t# overflow check long" %}
15553   ins_cost(INSN_COST);
15554   ins_encode %{
15555     __ cmp(zr, $op1$$Register);
15556   %}
15557 
15558   ins_pipe(icmp_reg_imm);
15559 %}
15560 
15561 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15562 %{
15563   match(Set cr (OverflowMulI op1 op2));
15564 
15565   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15566             "cmp   rscratch1, rscratch1, sxtw\n\t"
15567             "movw  rscratch1, #0x80000000\n\t"
15568             "cselw rscratch1, rscratch1, zr, NE\n\t"
15569             "cmpw  rscratch1, #1" %}
15570   ins_cost(5 * INSN_COST);
15571   ins_encode %{
15572     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15573     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15574     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15575     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15576     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15577   %}
15578 
15579   ins_pipe(pipe_slow);
15580 %}
15581 
15582 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15583 %{
15584   match(If cmp (OverflowMulI op1 op2));
15585   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15586             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15587   effect(USE labl, KILL cr);
15588 
15589   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15590             "cmp   rscratch1, rscratch1, sxtw\n\t"
15591             "b$cmp   $labl" %}
15592   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15593   ins_encode %{
15594     Label* L = $labl$$label;
15595     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15596     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15597     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15598     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15599   %}
15600 
15601   ins_pipe(pipe_serial);
15602 %}
15603 
15604 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15605 %{
15606   match(Set cr (OverflowMulL op1 op2));
15607 
15608   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15609             "smulh rscratch2, $op1, $op2\n\t"
15610             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15611             "movw  rscratch1, #0x80000000\n\t"
15612             "cselw rscratch1, rscratch1, zr, NE\n\t"
15613             "cmpw  rscratch1, #1" %}
15614   ins_cost(6 * INSN_COST);
15615   ins_encode %{
15616     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15617     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15618     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15619     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15620     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15621     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15622   %}
15623 
15624   ins_pipe(pipe_slow);
15625 %}
15626 
15627 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15628 %{
15629   match(If cmp (OverflowMulL op1 op2));
15630   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15631             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15632   effect(USE labl, KILL cr);
15633 
15634   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15635             "smulh rscratch2, $op1, $op2\n\t"
15636             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15637             "b$cmp $labl" %}
15638   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15639   ins_encode %{
15640     Label* L = $labl$$label;
15641     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15642     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15643     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15644     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15645     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15646   %}
15647 
15648   ins_pipe(pipe_serial);
15649 %}
15650 
15651 // ============================================================================
15652 // Compare Instructions
15653 
15654 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15655 %{
15656   match(Set cr (CmpI op1 op2));
15657 
15658   effect(DEF cr, USE op1, USE op2);
15659 
15660   ins_cost(INSN_COST);
15661   format %{ "cmpw  $op1, $op2" %}
15662 
15663   ins_encode(aarch64_enc_cmpw(op1, op2));
15664 
15665   ins_pipe(icmp_reg_reg);
15666 %}
15667 
15668 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15669 %{
15670   match(Set cr (CmpI op1 zero));
15671 
15672   effect(DEF cr, USE op1);
15673 
15674   ins_cost(INSN_COST);
15675   format %{ "cmpw $op1, 0" %}
15676 
15677   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15678 
15679   ins_pipe(icmp_reg_imm);
15680 %}
15681 
15682 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15683 %{
15684   match(Set cr (CmpI op1 op2));
15685 
15686   effect(DEF cr, USE op1);
15687 
15688   ins_cost(INSN_COST);
15689   format %{ "cmpw  $op1, $op2" %}
15690 
15691   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15692 
15693   ins_pipe(icmp_reg_imm);
15694 %}
15695 
15696 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15697 %{
15698   match(Set cr (CmpI op1 op2));
15699 
15700   effect(DEF cr, USE op1);
15701 
15702   ins_cost(INSN_COST * 2);
15703   format %{ "cmpw  $op1, $op2" %}
15704 
15705   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15706 
15707   ins_pipe(icmp_reg_imm);
15708 %}
15709 
15710 // Unsigned compare Instructions; really, same as signed compare
15711 // except it should only be used to feed an If or a CMovI which takes a
15712 // cmpOpU.
15713 
15714 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15715 %{
15716   match(Set cr (CmpU op1 op2));
15717 
15718   effect(DEF cr, USE op1, USE op2);
15719 
15720   ins_cost(INSN_COST);
15721   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15722 
15723   ins_encode(aarch64_enc_cmpw(op1, op2));
15724 
15725   ins_pipe(icmp_reg_reg);
15726 %}
15727 
15728 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15729 %{
15730   match(Set cr (CmpU op1 zero));
15731 
15732   effect(DEF cr, USE op1);
15733 
15734   ins_cost(INSN_COST);
15735   format %{ "cmpw $op1, #0\t# unsigned" %}
15736 
15737   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15738 
15739   ins_pipe(icmp_reg_imm);
15740 %}
15741 
15742 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15743 %{
15744   match(Set cr (CmpU op1 op2));
15745 
15746   effect(DEF cr, USE op1);
15747 
15748   ins_cost(INSN_COST);
15749   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15750 
15751   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15752 
15753   ins_pipe(icmp_reg_imm);
15754 %}
15755 
15756 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15757 %{
15758   match(Set cr (CmpU op1 op2));
15759 
15760   effect(DEF cr, USE op1);
15761 
15762   ins_cost(INSN_COST * 2);
15763   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15764 
15765   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15766 
15767   ins_pipe(icmp_reg_imm);
15768 %}
15769 
15770 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15771 %{
15772   match(Set cr (CmpL op1 op2));
15773 
15774   effect(DEF cr, USE op1, USE op2);
15775 
15776   ins_cost(INSN_COST);
15777   format %{ "cmp  $op1, $op2" %}
15778 
15779   ins_encode(aarch64_enc_cmp(op1, op2));
15780 
15781   ins_pipe(icmp_reg_reg);
15782 %}
15783 
15784 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15785 %{
15786   match(Set cr (CmpL op1 zero));
15787 
15788   effect(DEF cr, USE op1);
15789 
15790   ins_cost(INSN_COST);
15791   format %{ "tst  $op1" %}
15792 
15793   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15794 
15795   ins_pipe(icmp_reg_imm);
15796 %}
15797 
15798 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15799 %{
15800   match(Set cr (CmpL op1 op2));
15801 
15802   effect(DEF cr, USE op1);
15803 
15804   ins_cost(INSN_COST);
15805   format %{ "cmp  $op1, $op2" %}
15806 
15807   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15808 
15809   ins_pipe(icmp_reg_imm);
15810 %}
15811 
15812 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15813 %{
15814   match(Set cr (CmpL op1 op2));
15815 
15816   effect(DEF cr, USE op1);
15817 
15818   ins_cost(INSN_COST * 2);
15819   format %{ "cmp  $op1, $op2" %}
15820 
15821   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15822 
15823   ins_pipe(icmp_reg_imm);
15824 %}
15825 
15826 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15827 %{
15828   match(Set cr (CmpUL op1 op2));
15829 
15830   effect(DEF cr, USE op1, USE op2);
15831 
15832   ins_cost(INSN_COST);
15833   format %{ "cmp  $op1, $op2" %}
15834 
15835   ins_encode(aarch64_enc_cmp(op1, op2));
15836 
15837   ins_pipe(icmp_reg_reg);
15838 %}
15839 
15840 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15841 %{
15842   match(Set cr (CmpUL op1 zero));
15843 
15844   effect(DEF cr, USE op1);
15845 
15846   ins_cost(INSN_COST);
15847   format %{ "tst  $op1" %}
15848 
15849   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15850 
15851   ins_pipe(icmp_reg_imm);
15852 %}
15853 
15854 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15855 %{
15856   match(Set cr (CmpUL op1 op2));
15857 
15858   effect(DEF cr, USE op1);
15859 
15860   ins_cost(INSN_COST);
15861   format %{ "cmp  $op1, $op2" %}
15862 
15863   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15864 
15865   ins_pipe(icmp_reg_imm);
15866 %}
15867 
15868 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15869 %{
15870   match(Set cr (CmpUL op1 op2));
15871 
15872   effect(DEF cr, USE op1);
15873 
15874   ins_cost(INSN_COST * 2);
15875   format %{ "cmp  $op1, $op2" %}
15876 
15877   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15878 
15879   ins_pipe(icmp_reg_imm);
15880 %}
15881 
15882 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15883 %{
15884   match(Set cr (CmpP op1 op2));
15885 
15886   effect(DEF cr, USE op1, USE op2);
15887 
15888   ins_cost(INSN_COST);
15889   format %{ "cmp  $op1, $op2\t // ptr" %}
15890 
15891   ins_encode(aarch64_enc_cmpp(op1, op2));
15892 
15893   ins_pipe(icmp_reg_reg);
15894 %}
15895 
15896 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15897 %{
15898   match(Set cr (CmpN op1 op2));
15899 
15900   effect(DEF cr, USE op1, USE op2);
15901 
15902   ins_cost(INSN_COST);
15903   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15904 
15905   ins_encode(aarch64_enc_cmpn(op1, op2));
15906 
15907   ins_pipe(icmp_reg_reg);
15908 %}
15909 
15910 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15911 %{
15912   match(Set cr (CmpP op1 zero));
15913 
15914   effect(DEF cr, USE op1, USE zero);
15915 
15916   ins_cost(INSN_COST);
15917   format %{ "cmp  $op1, 0\t // ptr" %}
15918 
15919   ins_encode(aarch64_enc_testp(op1));
15920 
15921   ins_pipe(icmp_reg_imm);
15922 %}
15923 
15924 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15925 %{
15926   match(Set cr (CmpN op1 zero));
15927 
15928   effect(DEF cr, USE op1, USE zero);
15929 
15930   ins_cost(INSN_COST);
15931   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15932 
15933   ins_encode(aarch64_enc_testn(op1));
15934 
15935   ins_pipe(icmp_reg_imm);
15936 %}
15937 
15938 // FP comparisons
15939 //
15940 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15941 // using normal cmpOp. See declaration of rFlagsReg for details.
15942 
15943 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15944 %{
15945   match(Set cr (CmpF src1 src2));
15946 
15947   ins_cost(3 * INSN_COST);
15948   format %{ "fcmps $src1, $src2" %}
15949 
15950   ins_encode %{
15951     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15952   %}
15953 
15954   ins_pipe(pipe_class_compare);
15955 %}
15956 
15957 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15958 %{
15959   match(Set cr (CmpF src1 src2));
15960 
15961   ins_cost(3 * INSN_COST);
15962   format %{ "fcmps $src1, 0.0" %}
15963 
15964   ins_encode %{
15965     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15966   %}
15967 
15968   ins_pipe(pipe_class_compare);
15969 %}
15970 // FROM HERE
15971 
15972 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15973 %{
15974   match(Set cr (CmpD src1 src2));
15975 
15976   ins_cost(3 * INSN_COST);
15977   format %{ "fcmpd $src1, $src2" %}
15978 
15979   ins_encode %{
15980     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15981   %}
15982 
15983   ins_pipe(pipe_class_compare);
15984 %}
15985 
15986 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15987 %{
15988   match(Set cr (CmpD src1 src2));
15989 
15990   ins_cost(3 * INSN_COST);
15991   format %{ "fcmpd $src1, 0.0" %}
15992 
15993   ins_encode %{
15994     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15995   %}
15996 
15997   ins_pipe(pipe_class_compare);
15998 %}
15999 
16000 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16001 %{
16002   match(Set dst (CmpF3 src1 src2));
16003   effect(KILL cr);
16004 
16005   ins_cost(5 * INSN_COST);
16006   format %{ "fcmps $src1, $src2\n\t"
16007             "csinvw($dst, zr, zr, eq\n\t"
16008             "csnegw($dst, $dst, $dst, lt)"
16009   %}
16010 
16011   ins_encode %{
16012     Label done;
16013     FloatRegister s1 = as_FloatRegister($src1$$reg);
16014     FloatRegister s2 = as_FloatRegister($src2$$reg);
16015     Register d = as_Register($dst$$reg);
16016     __ fcmps(s1, s2);
16017     // installs 0 if EQ else -1
16018     __ csinvw(d, zr, zr, Assembler::EQ);
16019     // keeps -1 if less or unordered else installs 1
16020     __ csnegw(d, d, d, Assembler::LT);
16021     __ bind(done);
16022   %}
16023 
16024   ins_pipe(pipe_class_default);
16025 
16026 %}
16027 
16028 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16029 %{
16030   match(Set dst (CmpD3 src1 src2));
16031   effect(KILL cr);
16032 
16033   ins_cost(5 * INSN_COST);
16034   format %{ "fcmpd $src1, $src2\n\t"
16035             "csinvw($dst, zr, zr, eq\n\t"
16036             "csnegw($dst, $dst, $dst, lt)"
16037   %}
16038 
16039   ins_encode %{
16040     Label done;
16041     FloatRegister s1 = as_FloatRegister($src1$$reg);
16042     FloatRegister s2 = as_FloatRegister($src2$$reg);
16043     Register d = as_Register($dst$$reg);
16044     __ fcmpd(s1, s2);
16045     // installs 0 if EQ else -1
16046     __ csinvw(d, zr, zr, Assembler::EQ);
16047     // keeps -1 if less or unordered else installs 1
16048     __ csnegw(d, d, d, Assembler::LT);
16049     __ bind(done);
16050   %}
16051   ins_pipe(pipe_class_default);
16052 
16053 %}
16054 
16055 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16056 %{
16057   match(Set dst (CmpF3 src1 zero));
16058   effect(KILL cr);
16059 
16060   ins_cost(5 * INSN_COST);
16061   format %{ "fcmps $src1, 0.0\n\t"
16062             "csinvw($dst, zr, zr, eq\n\t"
16063             "csnegw($dst, $dst, $dst, lt)"
16064   %}
16065 
16066   ins_encode %{
16067     Label done;
16068     FloatRegister s1 = as_FloatRegister($src1$$reg);
16069     Register d = as_Register($dst$$reg);
16070     __ fcmps(s1, 0.0);
16071     // installs 0 if EQ else -1
16072     __ csinvw(d, zr, zr, Assembler::EQ);
16073     // keeps -1 if less or unordered else installs 1
16074     __ csnegw(d, d, d, Assembler::LT);
16075     __ bind(done);
16076   %}
16077 
16078   ins_pipe(pipe_class_default);
16079 
16080 %}
16081 
16082 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16083 %{
16084   match(Set dst (CmpD3 src1 zero));
16085   effect(KILL cr);
16086 
16087   ins_cost(5 * INSN_COST);
16088   format %{ "fcmpd $src1, 0.0\n\t"
16089             "csinvw($dst, zr, zr, eq\n\t"
16090             "csnegw($dst, $dst, $dst, lt)"
16091   %}
16092 
16093   ins_encode %{
16094     Label done;
16095     FloatRegister s1 = as_FloatRegister($src1$$reg);
16096     Register d = as_Register($dst$$reg);
16097     __ fcmpd(s1, 0.0);
16098     // installs 0 if EQ else -1
16099     __ csinvw(d, zr, zr, Assembler::EQ);
16100     // keeps -1 if less or unordered else installs 1
16101     __ csnegw(d, d, d, Assembler::LT);
16102     __ bind(done);
16103   %}
16104   ins_pipe(pipe_class_default);
16105 
16106 %}
16107 
16108 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16109 %{
16110   match(Set dst (CmpLTMask p q));
16111   effect(KILL cr);
16112 
16113   ins_cost(3 * INSN_COST);
16114 
16115   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16116             "csetw $dst, lt\n\t"
16117             "subw $dst, zr, $dst"
16118   %}
16119 
16120   ins_encode %{
16121     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16122     __ csetw(as_Register($dst$$reg), Assembler::LT);
16123     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16124   %}
16125 
16126   ins_pipe(ialu_reg_reg);
16127 %}
16128 
16129 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16130 %{
16131   match(Set dst (CmpLTMask src zero));
16132   effect(KILL cr);
16133 
16134   ins_cost(INSN_COST);
16135 
16136   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16137 
16138   ins_encode %{
16139     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16140   %}
16141 
16142   ins_pipe(ialu_reg_shift);
16143 %}
16144 
16145 // ============================================================================
16146 // Max and Min
16147 
16148 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16149 %{
16150   effect( DEF dst, USE src1, USE src2, USE cr );
16151 
16152   ins_cost(INSN_COST * 2);
16153   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
16154 
16155   ins_encode %{
16156     __ cselw(as_Register($dst$$reg),
16157              as_Register($src1$$reg),
16158              as_Register($src2$$reg),
16159              Assembler::LT);
16160   %}
16161 
16162   ins_pipe(icond_reg_reg);
16163 %}
16164 
16165 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16166 %{
16167   match(Set dst (MinI src1 src2));
16168   ins_cost(INSN_COST * 3);
16169 
16170   expand %{
16171     rFlagsReg cr;
16172     compI_reg_reg(cr, src1, src2);
16173     cmovI_reg_reg_lt(dst, src1, src2, cr);
16174   %}
16175 
16176 %}
16177 // FROM HERE
16178 
16179 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16180 %{
16181   effect( DEF dst, USE src1, USE src2, USE cr );
16182 
16183   ins_cost(INSN_COST * 2);
16184   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
16185 
16186   ins_encode %{
16187     __ cselw(as_Register($dst$$reg),
16188              as_Register($src1$$reg),
16189              as_Register($src2$$reg),
16190              Assembler::GT);
16191   %}
16192 
16193   ins_pipe(icond_reg_reg);
16194 %}
16195 
16196 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16197 %{
16198   match(Set dst (MaxI src1 src2));
16199   ins_cost(INSN_COST * 3);
16200   expand %{
16201     rFlagsReg cr;
16202     compI_reg_reg(cr, src1, src2);
16203     cmovI_reg_reg_gt(dst, src1, src2, cr);
16204   %}
16205 %}
16206 
16207 // ============================================================================
16208 // Branch Instructions
16209 
16210 // Direct Branch.
16211 instruct branch(label lbl)
16212 %{
16213   match(Goto);
16214 
16215   effect(USE lbl);
16216 
16217   ins_cost(BRANCH_COST);
16218   format %{ "b  $lbl" %}
16219 
16220   ins_encode(aarch64_enc_b(lbl));
16221 
16222   ins_pipe(pipe_branch);
16223 %}
16224 
16225 // Conditional Near Branch
16226 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16227 %{
16228   // Same match rule as `branchConFar'.
16229   match(If cmp cr);
16230 
16231   effect(USE lbl);
16232 
16233   ins_cost(BRANCH_COST);
16234   // If set to 1 this indicates that the current instruction is a
16235   // short variant of a long branch. This avoids using this
16236   // instruction in first-pass matching. It will then only be used in
16237   // the `Shorten_branches' pass.
16238   // ins_short_branch(1);
16239   format %{ "b$cmp  $lbl" %}
16240 
16241   ins_encode(aarch64_enc_br_con(cmp, lbl));
16242 
16243   ins_pipe(pipe_branch_cond);
16244 %}
16245 
16246 // Conditional Near Branch Unsigned
16247 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16248 %{
16249   // Same match rule as `branchConFar'.
16250   match(If cmp cr);
16251 
16252   effect(USE lbl);
16253 
16254   ins_cost(BRANCH_COST);
16255   // If set to 1 this indicates that the current instruction is a
16256   // short variant of a long branch. This avoids using this
16257   // instruction in first-pass matching. It will then only be used in
16258   // the `Shorten_branches' pass.
16259   // ins_short_branch(1);
16260   format %{ "b$cmp  $lbl\t# unsigned" %}
16261 
16262   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16263 
16264   ins_pipe(pipe_branch_cond);
16265 %}
16266 
16267 // Make use of CBZ and CBNZ.  These instructions, as well as being
16268 // shorter than (cmp; branch), have the additional benefit of not
16269 // killing the flags.
16270 
16271 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16272   match(If cmp (CmpI op1 op2));
16273   effect(USE labl);
16274 
16275   ins_cost(BRANCH_COST);
16276   format %{ "cbw$cmp   $op1, $labl" %}
16277   ins_encode %{
16278     Label* L = $labl$$label;
16279     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16280     if (cond == Assembler::EQ)
16281       __ cbzw($op1$$Register, *L);
16282     else
16283       __ cbnzw($op1$$Register, *L);
16284   %}
16285   ins_pipe(pipe_cmp_branch);
16286 %}
16287 
16288 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16289   match(If cmp (CmpL op1 op2));
16290   effect(USE labl);
16291 
16292   ins_cost(BRANCH_COST);
16293   format %{ "cb$cmp   $op1, $labl" %}
16294   ins_encode %{
16295     Label* L = $labl$$label;
16296     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16297     if (cond == Assembler::EQ)
16298       __ cbz($op1$$Register, *L);
16299     else
16300       __ cbnz($op1$$Register, *L);
16301   %}
16302   ins_pipe(pipe_cmp_branch);
16303 %}
16304 
16305 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16306   match(If cmp (CmpP op1 op2));
16307   effect(USE labl);
16308 
16309   ins_cost(BRANCH_COST);
16310   format %{ "cb$cmp   $op1, $labl" %}
16311   ins_encode %{
16312     Label* L = $labl$$label;
16313     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16314     if (cond == Assembler::EQ)
16315       __ cbz($op1$$Register, *L);
16316     else
16317       __ cbnz($op1$$Register, *L);
16318   %}
16319   ins_pipe(pipe_cmp_branch);
16320 %}
16321 
16322 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16323   match(If cmp (CmpN op1 op2));
16324   effect(USE labl);
16325 
16326   ins_cost(BRANCH_COST);
16327   format %{ "cbw$cmp   $op1, $labl" %}
16328   ins_encode %{
16329     Label* L = $labl$$label;
16330     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16331     if (cond == Assembler::EQ)
16332       __ cbzw($op1$$Register, *L);
16333     else
16334       __ cbnzw($op1$$Register, *L);
16335   %}
16336   ins_pipe(pipe_cmp_branch);
16337 %}
16338 
16339 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16340   match(If cmp (CmpP (DecodeN oop) zero));
16341   effect(USE labl);
16342 
16343   ins_cost(BRANCH_COST);
16344   format %{ "cb$cmp   $oop, $labl" %}
16345   ins_encode %{
16346     Label* L = $labl$$label;
16347     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16348     if (cond == Assembler::EQ)
16349       __ cbzw($oop$$Register, *L);
16350     else
16351       __ cbnzw($oop$$Register, *L);
16352   %}
16353   ins_pipe(pipe_cmp_branch);
16354 %}
16355 
16356 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16357   match(If cmp (CmpU op1 op2));
16358   effect(USE labl);
16359 
16360   ins_cost(BRANCH_COST);
16361   format %{ "cbw$cmp   $op1, $labl" %}
16362   ins_encode %{
16363     Label* L = $labl$$label;
16364     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16365     if (cond == Assembler::EQ || cond == Assembler::LS)
16366       __ cbzw($op1$$Register, *L);
16367     else
16368       __ cbnzw($op1$$Register, *L);
16369   %}
16370   ins_pipe(pipe_cmp_branch);
16371 %}
16372 
16373 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16374   match(If cmp (CmpUL op1 op2));
16375   effect(USE labl);
16376 
16377   ins_cost(BRANCH_COST);
16378   format %{ "cb$cmp   $op1, $labl" %}
16379   ins_encode %{
16380     Label* L = $labl$$label;
16381     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16382     if (cond == Assembler::EQ || cond == Assembler::LS)
16383       __ cbz($op1$$Register, *L);
16384     else
16385       __ cbnz($op1$$Register, *L);
16386   %}
16387   ins_pipe(pipe_cmp_branch);
16388 %}
16389 
16390 // Test bit and Branch
16391 
16392 // Patterns for short (< 32KiB) variants
16393 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16394   match(If cmp (CmpL op1 op2));
16395   effect(USE labl);
16396 
16397   ins_cost(BRANCH_COST);
16398   format %{ "cb$cmp   $op1, $labl # long" %}
16399   ins_encode %{
16400     Label* L = $labl$$label;
16401     Assembler::Condition cond =
16402       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16403     __ tbr(cond, $op1$$Register, 63, *L);
16404   %}
16405   ins_pipe(pipe_cmp_branch);
16406   ins_short_branch(1);
16407 %}
16408 
16409 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16410   match(If cmp (CmpI op1 op2));
16411   effect(USE labl);
16412 
16413   ins_cost(BRANCH_COST);
16414   format %{ "cb$cmp   $op1, $labl # int" %}
16415   ins_encode %{
16416     Label* L = $labl$$label;
16417     Assembler::Condition cond =
16418       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16419     __ tbr(cond, $op1$$Register, 31, *L);
16420   %}
16421   ins_pipe(pipe_cmp_branch);
16422   ins_short_branch(1);
16423 %}
16424 
16425 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16426   match(If cmp (CmpL (AndL op1 op2) op3));
16427   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16428   effect(USE labl);
16429 
16430   ins_cost(BRANCH_COST);
16431   format %{ "tb$cmp   $op1, $op2, $labl" %}
16432   ins_encode %{
16433     Label* L = $labl$$label;
16434     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16435     int bit = exact_log2_long($op2$$constant);
16436     __ tbr(cond, $op1$$Register, bit, *L);
16437   %}
16438   ins_pipe(pipe_cmp_branch);
16439   ins_short_branch(1);
16440 %}
16441 
16442 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16443   match(If cmp (CmpI (AndI op1 op2) op3));
16444   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16445   effect(USE labl);
16446 
16447   ins_cost(BRANCH_COST);
16448   format %{ "tb$cmp   $op1, $op2, $labl" %}
16449   ins_encode %{
16450     Label* L = $labl$$label;
16451     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16452     int bit = exact_log2((juint)$op2$$constant);
16453     __ tbr(cond, $op1$$Register, bit, *L);
16454   %}
16455   ins_pipe(pipe_cmp_branch);
16456   ins_short_branch(1);
16457 %}
16458 
16459 // And far variants
16460 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16461   match(If cmp (CmpL op1 op2));
16462   effect(USE labl);
16463 
16464   ins_cost(BRANCH_COST);
16465   format %{ "cb$cmp   $op1, $labl # long" %}
16466   ins_encode %{
16467     Label* L = $labl$$label;
16468     Assembler::Condition cond =
16469       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16470     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16471   %}
16472   ins_pipe(pipe_cmp_branch);
16473 %}
16474 
16475 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16476   match(If cmp (CmpI op1 op2));
16477   effect(USE labl);
16478 
16479   ins_cost(BRANCH_COST);
16480   format %{ "cb$cmp   $op1, $labl # int" %}
16481   ins_encode %{
16482     Label* L = $labl$$label;
16483     Assembler::Condition cond =
16484       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16485     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16486   %}
16487   ins_pipe(pipe_cmp_branch);
16488 %}
16489 
16490 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16491   match(If cmp (CmpL (AndL op1 op2) op3));
16492   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16493   effect(USE labl);
16494 
16495   ins_cost(BRANCH_COST);
16496   format %{ "tb$cmp   $op1, $op2, $labl" %}
16497   ins_encode %{
16498     Label* L = $labl$$label;
16499     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16500     int bit = exact_log2_long($op2$$constant);
16501     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16502   %}
16503   ins_pipe(pipe_cmp_branch);
16504 %}
16505 
16506 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16507   match(If cmp (CmpI (AndI op1 op2) op3));
16508   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16509   effect(USE labl);
16510 
16511   ins_cost(BRANCH_COST);
16512   format %{ "tb$cmp   $op1, $op2, $labl" %}
16513   ins_encode %{
16514     Label* L = $labl$$label;
16515     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16516     int bit = exact_log2((juint)$op2$$constant);
16517     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16518   %}
16519   ins_pipe(pipe_cmp_branch);
16520 %}
16521 
16522 // Test bits
16523 
16524 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16525   match(Set cr (CmpL (AndL op1 op2) op3));
16526   predicate(Assembler::operand_valid_for_logical_immediate
16527             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16528 
16529   ins_cost(INSN_COST);
16530   format %{ "tst $op1, $op2 # long" %}
16531   ins_encode %{
16532     __ tst($op1$$Register, $op2$$constant);
16533   %}
16534   ins_pipe(ialu_reg_reg);
16535 %}
16536 
16537 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16538   match(Set cr (CmpI (AndI op1 op2) op3));
16539   predicate(Assembler::operand_valid_for_logical_immediate
16540             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16541 
16542   ins_cost(INSN_COST);
16543   format %{ "tst $op1, $op2 # int" %}
16544   ins_encode %{
16545     __ tstw($op1$$Register, $op2$$constant);
16546   %}
16547   ins_pipe(ialu_reg_reg);
16548 %}
16549 
16550 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16551   match(Set cr (CmpL (AndL op1 op2) op3));
16552 
16553   ins_cost(INSN_COST);
16554   format %{ "tst $op1, $op2 # long" %}
16555   ins_encode %{
16556     __ tst($op1$$Register, $op2$$Register);
16557   %}
16558   ins_pipe(ialu_reg_reg);
16559 %}
16560 
16561 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16562   match(Set cr (CmpI (AndI op1 op2) op3));
16563 
16564   ins_cost(INSN_COST);
16565   format %{ "tstw $op1, $op2 # int" %}
16566   ins_encode %{
16567     __ tstw($op1$$Register, $op2$$Register);
16568   %}
16569   ins_pipe(ialu_reg_reg);
16570 %}
16571 
16572 
16573 // Conditional Far Branch
16574 // Conditional Far Branch Unsigned
16575 // TODO: fixme
16576 
16577 // counted loop end branch near
16578 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16579 %{
16580   match(CountedLoopEnd cmp cr);
16581 
16582   effect(USE lbl);
16583 
16584   ins_cost(BRANCH_COST);
16585   // short variant.
16586   // ins_short_branch(1);
16587   format %{ "b$cmp $lbl \t// counted loop end" %}
16588 
16589   ins_encode(aarch64_enc_br_con(cmp, lbl));
16590 
16591   ins_pipe(pipe_branch);
16592 %}
16593 
16594 // counted loop end branch near Unsigned
16595 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16596 %{
16597   match(CountedLoopEnd cmp cr);
16598 
16599   effect(USE lbl);
16600 
16601   ins_cost(BRANCH_COST);
16602   // short variant.
16603   // ins_short_branch(1);
16604   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16605 
16606   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16607 
16608   ins_pipe(pipe_branch);
16609 %}
16610 
16611 // counted loop end branch far
16612 // counted loop end branch far unsigned
16613 // TODO: fixme
16614 
16615 // ============================================================================
16616 // inlined locking and unlocking
16617 
16618 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16619 %{
16620   match(Set cr (FastLock object box));
16621   effect(TEMP tmp, TEMP tmp2);
16622 
16623   // TODO
16624   // identify correct cost
16625   ins_cost(5 * INSN_COST);
16626   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16627 
16628   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16629 
16630   ins_pipe(pipe_serial);
16631 %}
16632 
16633 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16634 %{
16635   match(Set cr (FastUnlock object box));
16636   effect(TEMP tmp, TEMP tmp2);
16637 
16638   ins_cost(5 * INSN_COST);
16639   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16640 
16641   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16642 
16643   ins_pipe(pipe_serial);
16644 %}
16645 
16646 
16647 // ============================================================================
16648 // Safepoint Instructions
16649 
16650 // TODO
16651 // provide a near and far version of this code
16652 
16653 instruct safePoint(rFlagsReg cr, iRegP poll)
16654 %{
16655   match(SafePoint poll);
16656   effect(KILL cr);
16657 
16658   format %{
16659     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16660   %}
16661   ins_encode %{
16662     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16663   %}
16664   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16665 %}
16666 
16667 
16668 // ============================================================================
16669 // Procedure Call/Return Instructions
16670 
16671 // Call Java Static Instruction
16672 
16673 instruct CallStaticJavaDirect(method meth)
16674 %{
16675   match(CallStaticJava);
16676 
16677   effect(USE meth);
16678 
16679   ins_cost(CALL_COST);
16680 
16681   format %{ "call,static $meth \t// ==> " %}
16682 
16683   ins_encode(aarch64_enc_java_static_call(meth),
16684              aarch64_enc_call_epilog);
16685 
16686   ins_pipe(pipe_class_call);
16687 %}
16688 
16689 // TO HERE
16690 
16691 // Call Java Dynamic Instruction
16692 instruct CallDynamicJavaDirect(method meth)
16693 %{
16694   match(CallDynamicJava);
16695 
16696   effect(USE meth);
16697 
16698   ins_cost(CALL_COST);
16699 
16700   format %{ "CALL,dynamic $meth \t// ==> " %}
16701 
16702   ins_encode(aarch64_enc_java_dynamic_call(meth),
16703              aarch64_enc_call_epilog);
16704 
16705   ins_pipe(pipe_class_call);
16706 %}
16707 
16708 // Call Runtime Instruction
16709 
16710 instruct CallRuntimeDirect(method meth)
16711 %{
16712   match(CallRuntime);
16713 
16714   effect(USE meth);
16715 
16716   ins_cost(CALL_COST);
16717 
16718   format %{ "CALL, runtime $meth" %}
16719 
16720   ins_encode( aarch64_enc_java_to_runtime(meth) );
16721 
16722   ins_pipe(pipe_class_call);
16723 %}
16724 
16725 // Call Runtime Instruction
16726 
16727 instruct CallLeafDirect(method meth)
16728 %{
16729   match(CallLeaf);
16730 
16731   effect(USE meth);
16732 
16733   ins_cost(CALL_COST);
16734 
16735   format %{ "CALL, runtime leaf $meth" %}
16736 
16737   ins_encode( aarch64_enc_java_to_runtime(meth) );
16738 
16739   ins_pipe(pipe_class_call);
16740 %}
16741 
16742 // Call Runtime Instruction
16743 
16744 instruct CallLeafNoFPDirect(method meth)
16745 %{
16746   match(CallLeafNoFP);
16747 
16748   effect(USE meth);
16749 
16750   ins_cost(CALL_COST);
16751 
16752   format %{ "CALL, runtime leaf nofp $meth" %}
16753 
16754   ins_encode( aarch64_enc_java_to_runtime(meth) );
16755 
16756   ins_pipe(pipe_class_call);
16757 %}
16758 
16759 instruct CallNativeDirect(method meth)
16760 %{
16761   match(CallNative);
16762 
16763   effect(USE meth);
16764 
16765   ins_cost(CALL_COST);
16766 
16767   format %{ "CALL, native $meth" %}
16768 
16769   ins_encode( aarch64_enc_java_to_runtime(meth) );
16770 
16771   ins_pipe(pipe_class_call);
16772 %}
16773 
16774 // Tail Call; Jump from runtime stub to Java code.
16775 // Also known as an 'interprocedural jump'.
16776 // Target of jump will eventually return to caller.
16777 // TailJump below removes the return address.
16778 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16779 %{
16780   match(TailCall jump_target method_ptr);
16781 
16782   ins_cost(CALL_COST);
16783 
16784   format %{ "br $jump_target\t# $method_ptr holds method" %}
16785 
16786   ins_encode(aarch64_enc_tail_call(jump_target));
16787 
16788   ins_pipe(pipe_class_call);
16789 %}
16790 
16791 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16792 %{
16793   match(TailJump jump_target ex_oop);
16794 
16795   ins_cost(CALL_COST);
16796 
16797   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16798 
16799   ins_encode(aarch64_enc_tail_jmp(jump_target));
16800 
16801   ins_pipe(pipe_class_call);
16802 %}
16803 
16804 // Create exception oop: created by stack-crawling runtime code.
16805 // Created exception is now available to this handler, and is setup
16806 // just prior to jumping to this handler. No code emitted.
16807 // TODO check
16808 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16809 instruct CreateException(iRegP_R0 ex_oop)
16810 %{
16811   match(Set ex_oop (CreateEx));
16812 
16813   format %{ " -- \t// exception oop; no code emitted" %}
16814 
16815   size(0);
16816 
16817   ins_encode( /*empty*/ );
16818 
16819   ins_pipe(pipe_class_empty);
16820 %}
16821 
16822 // Rethrow exception: The exception oop will come in the first
16823 // argument position. Then JUMP (not call) to the rethrow stub code.
16824 instruct RethrowException() %{
16825   match(Rethrow);
16826   ins_cost(CALL_COST);
16827 
16828   format %{ "b rethrow_stub" %}
16829 
16830   ins_encode( aarch64_enc_rethrow() );
16831 
16832   ins_pipe(pipe_class_call);
16833 %}
16834 
16835 
16836 // Return Instruction
16837 // epilog node loads ret address into lr as part of frame pop
16838 instruct Ret()
16839 %{
16840   match(Return);
16841 
16842   format %{ "ret\t// return register" %}
16843 
16844   ins_encode( aarch64_enc_ret() );
16845 
16846   ins_pipe(pipe_branch);
16847 %}
16848 
16849 // Die now.
16850 instruct ShouldNotReachHere() %{
16851   match(Halt);
16852 
16853   ins_cost(CALL_COST);
16854   format %{ "ShouldNotReachHere" %}
16855 
16856   ins_encode %{
16857     if (is_reachable()) {
16858       __ stop(_halt_reason);
16859     }
16860   %}
16861 
16862   ins_pipe(pipe_class_default);
16863 %}
16864 
16865 // ============================================================================
16866 // Partial Subtype Check
16867 //
16868 // superklass array for an instance of the superklass.  Set a hidden
16869 // internal cache on a hit (cache is checked with exposed code in
16870 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16871 // encoding ALSO sets flags.
16872 
16873 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16874 %{
16875   match(Set result (PartialSubtypeCheck sub super));
16876   effect(KILL cr, KILL temp);
16877 
16878   ins_cost(1100);  // slightly larger than the next version
16879   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16880 
16881   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16882 
16883   opcode(0x1); // Force zero of result reg on hit
16884 
16885   ins_pipe(pipe_class_memory);
16886 %}
16887 
16888 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16889 %{
16890   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16891   effect(KILL temp, KILL result);
16892 
16893   ins_cost(1100);  // slightly larger than the next version
16894   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16895 
16896   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16897 
16898   opcode(0x0); // Don't zero result reg on hit
16899 
16900   ins_pipe(pipe_class_memory);
16901 %}
16902 
16903 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16904                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16905 %{
16906   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16907   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16908   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16909 
16910   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16911   ins_encode %{
16912     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16913     __ string_compare($str1$$Register, $str2$$Register,
16914                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16915                       $tmp1$$Register, $tmp2$$Register,
16916                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16917   %}
16918   ins_pipe(pipe_class_memory);
16919 %}
16920 
16921 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16922                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16923 %{
16924   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16925   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16926   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16927 
16928   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16929   ins_encode %{
16930     __ string_compare($str1$$Register, $str2$$Register,
16931                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16932                       $tmp1$$Register, $tmp2$$Register,
16933                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16934   %}
16935   ins_pipe(pipe_class_memory);
16936 %}
16937 
16938 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16939                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16940                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16941 %{
16942   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16943   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16944   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16945          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16946 
16947   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16948   ins_encode %{
16949     __ string_compare($str1$$Register, $str2$$Register,
16950                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16951                       $tmp1$$Register, $tmp2$$Register,
16952                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16953                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16954   %}
16955   ins_pipe(pipe_class_memory);
16956 %}
16957 
16958 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16959                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16960                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16961 %{
16962   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16963   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16964   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16965          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16966 
16967   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16968   ins_encode %{
16969     __ string_compare($str1$$Register, $str2$$Register,
16970                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16971                       $tmp1$$Register, $tmp2$$Register,
16972                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16973                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16974   %}
16975   ins_pipe(pipe_class_memory);
16976 %}
16977 
16978 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16979        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16980        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16981 %{
16982   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16983   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16984   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16985          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16986   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16987 
16988   ins_encode %{
16989     __ string_indexof($str1$$Register, $str2$$Register,
16990                       $cnt1$$Register, $cnt2$$Register,
16991                       $tmp1$$Register, $tmp2$$Register,
16992                       $tmp3$$Register, $tmp4$$Register,
16993                       $tmp5$$Register, $tmp6$$Register,
16994                       -1, $result$$Register, StrIntrinsicNode::UU);
16995   %}
16996   ins_pipe(pipe_class_memory);
16997 %}
16998 
16999 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17000        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17001        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17002 %{
17003   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17004   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17005   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17006          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17007   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
17008 
17009   ins_encode %{
17010     __ string_indexof($str1$$Register, $str2$$Register,
17011                       $cnt1$$Register, $cnt2$$Register,
17012                       $tmp1$$Register, $tmp2$$Register,
17013                       $tmp3$$Register, $tmp4$$Register,
17014                       $tmp5$$Register, $tmp6$$Register,
17015                       -1, $result$$Register, StrIntrinsicNode::LL);
17016   %}
17017   ins_pipe(pipe_class_memory);
17018 %}
17019 
17020 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17021        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17022        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17023 %{
17024   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17025   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17026   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17027          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17028   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17029 
17030   ins_encode %{
17031     __ string_indexof($str1$$Register, $str2$$Register,
17032                       $cnt1$$Register, $cnt2$$Register,
17033                       $tmp1$$Register, $tmp2$$Register,
17034                       $tmp3$$Register, $tmp4$$Register,
17035                       $tmp5$$Register, $tmp6$$Register,
17036                       -1, $result$$Register, StrIntrinsicNode::UL);
17037   %}
17038   ins_pipe(pipe_class_memory);
17039 %}
17040 
17041 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17042                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17043                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17044 %{
17045   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17046   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17047   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17048          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17049   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17050 
17051   ins_encode %{
17052     int icnt2 = (int)$int_cnt2$$constant;
17053     __ string_indexof($str1$$Register, $str2$$Register,
17054                       $cnt1$$Register, zr,
17055                       $tmp1$$Register, $tmp2$$Register,
17056                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17057                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17058   %}
17059   ins_pipe(pipe_class_memory);
17060 %}
17061 
17062 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17063                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17064                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17065 %{
17066   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17067   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17068   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17069          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17070   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17071 
17072   ins_encode %{
17073     int icnt2 = (int)$int_cnt2$$constant;
17074     __ string_indexof($str1$$Register, $str2$$Register,
17075                       $cnt1$$Register, zr,
17076                       $tmp1$$Register, $tmp2$$Register,
17077                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17078                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17079   %}
17080   ins_pipe(pipe_class_memory);
17081 %}
17082 
17083 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17084                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17085                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17086 %{
17087   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17088   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17089   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17090          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17091   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17092 
17093   ins_encode %{
17094     int icnt2 = (int)$int_cnt2$$constant;
17095     __ string_indexof($str1$$Register, $str2$$Register,
17096                       $cnt1$$Register, zr,
17097                       $tmp1$$Register, $tmp2$$Register,
17098                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17099                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17100   %}
17101   ins_pipe(pipe_class_memory);
17102 %}
17103 
17104 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17105                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17106                              iRegINoSp tmp3, rFlagsReg cr)
17107 %{
17108   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17109   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17110   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17111          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17112 
17113   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17114 
17115   ins_encode %{
17116     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17117                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17118                            $tmp3$$Register);
17119   %}
17120   ins_pipe(pipe_class_memory);
17121 %}
17122 
17123 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17124                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17125                               iRegINoSp tmp3, rFlagsReg cr)
17126 %{
17127   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17128   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17129   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17130          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17131 
17132   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17133 
17134   ins_encode %{
17135     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17136                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17137                             $tmp3$$Register);
17138   %}
17139   ins_pipe(pipe_class_memory);
17140 %}
17141 
17142 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17143                         iRegI_R0 result, rFlagsReg cr)
17144 %{
17145   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17146   match(Set result (StrEquals (Binary str1 str2) cnt));
17147   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17148 
17149   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17150   ins_encode %{
17151     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17152     __ string_equals($str1$$Register, $str2$$Register,
17153                      $result$$Register, $cnt$$Register, 1);
17154   %}
17155   ins_pipe(pipe_class_memory);
17156 %}
17157 
17158 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17159                         iRegI_R0 result, rFlagsReg cr)
17160 %{
17161   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17162   match(Set result (StrEquals (Binary str1 str2) cnt));
17163   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17164 
17165   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17166   ins_encode %{
17167     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17168     __ string_equals($str1$$Register, $str2$$Register,
17169                      $result$$Register, $cnt$$Register, 2);
17170   %}
17171   ins_pipe(pipe_class_memory);
17172 %}
17173 
17174 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17175                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17176                        iRegP_R10 tmp, rFlagsReg cr)
17177 %{
17178   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17179   match(Set result (AryEq ary1 ary2));
17180   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17181 
17182   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17183   ins_encode %{
17184     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17185                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17186                                    $result$$Register, $tmp$$Register, 1);
17187     if (tpc == NULL) {
17188       ciEnv::current()->record_failure("CodeCache is full");
17189       return;
17190     }
17191   %}
17192   ins_pipe(pipe_class_memory);
17193 %}
17194 
17195 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17196                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17197                        iRegP_R10 tmp, rFlagsReg cr)
17198 %{
17199   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17200   match(Set result (AryEq ary1 ary2));
17201   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17202 
17203   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17204   ins_encode %{
17205     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17206                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17207                                    $result$$Register, $tmp$$Register, 2);
17208     if (tpc == NULL) {
17209       ciEnv::current()->record_failure("CodeCache is full");
17210       return;
17211     }
17212   %}
17213   ins_pipe(pipe_class_memory);
17214 %}
17215 
17216 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17217 %{
17218   match(Set result (CountPositives ary1 len));
17219   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17220   format %{ "count positives byte[] $ary1,$len -> $result" %}
17221   ins_encode %{
17222     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17223     if (tpc == NULL) {
17224       ciEnv::current()->record_failure("CodeCache is full");
17225       return;
17226     }
17227   %}
17228   ins_pipe( pipe_slow );
17229 %}
17230 
17231 // fast char[] to byte[] compression
17232 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17233                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17234                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17235                          iRegI_R0 result, rFlagsReg cr)
17236 %{
17237   match(Set result (StrCompressedCopy src (Binary dst len)));
17238   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17239          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17240 
17241   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
17242   ins_encode %{
17243     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17244                            $result$$Register,
17245                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17246                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17247   %}
17248   ins_pipe(pipe_slow);
17249 %}
17250 
17251 // fast byte[] to char[] inflation
17252 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17253                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17254 %{
17255   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17256   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17257 
17258   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17259   ins_encode %{
17260     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17261                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17262                                         $tmp3$$FloatRegister, $tmp4$$Register);
17263     if (tpc == NULL) {
17264       ciEnv::current()->record_failure("CodeCache is full");
17265       return;
17266     }
17267   %}
17268   ins_pipe(pipe_class_memory);
17269 %}
17270 
17271 // encode char[] to byte[] in ISO_8859_1
17272 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17273                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17274                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17275                           iRegI_R0 result, rFlagsReg cr)
17276 %{
17277   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17278   match(Set result (EncodeISOArray src (Binary dst len)));
17279   effect(USE_KILL src, USE_KILL dst, USE len,
17280          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17281 
17282   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17283   ins_encode %{
17284     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17285                         $result$$Register, false,
17286                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17287                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17288   %}
17289   ins_pipe(pipe_class_memory);
17290 %}
17291 
17292 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17293                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17294                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17295                             iRegI_R0 result, rFlagsReg cr)
17296 %{
17297   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17298   match(Set result (EncodeISOArray src (Binary dst len)));
17299   effect(USE_KILL src, USE_KILL dst, USE len,
17300          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17301 
17302   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17303   ins_encode %{
17304     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17305                         $result$$Register, true,
17306                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17307                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17308   %}
17309   ins_pipe(pipe_class_memory);
17310 %}
17311 
17312 // ============================================================================
17313 // This name is KNOWN by the ADLC and cannot be changed.
17314 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17315 // for this guy.
17316 instruct tlsLoadP(thread_RegP dst)
17317 %{
17318   match(Set dst (ThreadLocal));
17319 
17320   ins_cost(0);
17321 
17322   format %{ " -- \t// $dst=Thread::current(), empty" %}
17323 
17324   size(0);
17325 
17326   ins_encode( /*empty*/ );
17327 
17328   ins_pipe(pipe_class_empty);
17329 %}
17330 
17331 //----------PEEPHOLE RULES-----------------------------------------------------
17332 // These must follow all instruction definitions as they use the names
17333 // defined in the instructions definitions.
17334 //
17335 // peepmatch ( root_instr_name [preceding_instruction]* );
17336 //
17337 // peepconstraint %{
17338 // (instruction_number.operand_name relational_op instruction_number.operand_name
17339 //  [, ...] );
17340 // // instruction numbers are zero-based using left to right order in peepmatch
17341 //
17342 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17343 // // provide an instruction_number.operand_name for each operand that appears
17344 // // in the replacement instruction's match rule
17345 //
17346 // ---------VM FLAGS---------------------------------------------------------
17347 //
17348 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17349 //
17350 // Each peephole rule is given an identifying number starting with zero and
17351 // increasing by one in the order seen by the parser.  An individual peephole
17352 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17353 // on the command-line.
17354 //
17355 // ---------CURRENT LIMITATIONS----------------------------------------------
17356 //
17357 // Only match adjacent instructions in same basic block
17358 // Only equality constraints
17359 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17360 // Only one replacement instruction
17361 //
17362 // ---------EXAMPLE----------------------------------------------------------
17363 //
17364 // // pertinent parts of existing instructions in architecture description
17365 // instruct movI(iRegINoSp dst, iRegI src)
17366 // %{
17367 //   match(Set dst (CopyI src));
17368 // %}
17369 //
17370 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17371 // %{
17372 //   match(Set dst (AddI dst src));
17373 //   effect(KILL cr);
17374 // %}
17375 //
17376 // // Change (inc mov) to lea
17377 // peephole %{
17378 //   // increment preceded by register-register move
17379 //   peepmatch ( incI_iReg movI );
17380 //   // require that the destination register of the increment
17381 //   // match the destination register of the move
17382 //   peepconstraint ( 0.dst == 1.dst );
17383 //   // construct a replacement instruction that sets
17384 //   // the destination to ( move's source register + one )
17385 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17386 // %}
17387 //
17388 
17389 // Implementation no longer uses movX instructions since
17390 // machine-independent system no longer uses CopyX nodes.
17391 //
17392 // peephole
17393 // %{
17394 //   peepmatch (incI_iReg movI);
17395 //   peepconstraint (0.dst == 1.dst);
17396 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17397 // %}
17398 
17399 // peephole
17400 // %{
17401 //   peepmatch (decI_iReg movI);
17402 //   peepconstraint (0.dst == 1.dst);
17403 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17404 // %}
17405 
17406 // peephole
17407 // %{
17408 //   peepmatch (addI_iReg_imm movI);
17409 //   peepconstraint (0.dst == 1.dst);
17410 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17411 // %}
17412 
17413 // peephole
17414 // %{
17415 //   peepmatch (incL_iReg movL);
17416 //   peepconstraint (0.dst == 1.dst);
17417 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17418 // %}
17419 
17420 // peephole
17421 // %{
17422 //   peepmatch (decL_iReg movL);
17423 //   peepconstraint (0.dst == 1.dst);
17424 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17425 // %}
17426 
17427 // peephole
17428 // %{
17429 //   peepmatch (addL_iReg_imm movL);
17430 //   peepconstraint (0.dst == 1.dst);
17431 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17432 // %}
17433 
17434 // peephole
17435 // %{
17436 //   peepmatch (addP_iReg_imm movP);
17437 //   peepconstraint (0.dst == 1.dst);
17438 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17439 // %}
17440 
17441 // // Change load of spilled value to only a spill
17442 // instruct storeI(memory mem, iRegI src)
17443 // %{
17444 //   match(Set mem (StoreI mem src));
17445 // %}
17446 //
17447 // instruct loadI(iRegINoSp dst, memory mem)
17448 // %{
17449 //   match(Set dst (LoadI mem));
17450 // %}
17451 //
17452 
17453 //----------SMARTSPILL RULES---------------------------------------------------
17454 // These must follow all instruction definitions as they use the names
17455 // defined in the instructions definitions.
17456 
17457 // Local Variables:
17458 // mode: c++
17459 // End: