1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly accessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_codestub_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298   bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // Assert that the given node is not a variable shift.
 1315   bool assert_not_var_shift(const Node* n);
 1316 
 1317   // predicate controlling addressing modes
 1318   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1319 %}
 1320 
 1321 source %{
 1322 
 1323   // Derived RegMask with conditionally allocatable registers
 1324 
 1325   void PhaseOutput::pd_perform_mach_node_analysis() {
 1326   }
 1327 
 1328   int MachNode::pd_alignment_required() const {
 1329     return 1;
 1330   }
 1331 
 1332   int MachNode::compute_padding(int current_offset) const {
 1333     return 0;
 1334   }
 1335 
 1336   RegMask _ANY_REG32_mask;
 1337   RegMask _ANY_REG_mask;
 1338   RegMask _PTR_REG_mask;
 1339   RegMask _NO_SPECIAL_REG32_mask;
 1340   RegMask _NO_SPECIAL_REG_mask;
 1341   RegMask _NO_SPECIAL_PTR_REG_mask;
 1342 
 1343   void reg_mask_init() {
 1344     // We derive below RegMask(s) from the ones which are auto-generated from
 1345     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1346     // registers conditionally reserved.
 1347 
 1348     _ANY_REG32_mask = _ALL_REG32_mask;
 1349     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1350 
 1351     _ANY_REG_mask = _ALL_REG_mask;
 1352 
 1353     _PTR_REG_mask = _ALL_REG_mask;
 1354 
 1355     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1356     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1357 
 1358     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1362     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1363 
 1364     // r27 is not allocatable when compressed oops is on and heapbase is not
 1365     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1366     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1367       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1368       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1369       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1370     }
 1371 
 1372     // r29 is not allocatable when PreserveFramePointer is on
 1373     if (PreserveFramePointer) {
 1374       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1375       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1376       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1377     }
 1378   }
 1379 
 1380   // Optimizaton of volatile gets and puts
 1381   // -------------------------------------
 1382   //
 1383   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1384   // use to implement volatile reads and writes. For a volatile read
 1385   // we simply need
 1386   //
 1387   //   ldar<x>
 1388   //
 1389   // and for a volatile write we need
 1390   //
 1391   //   stlr<x>
 1392   //
 1393   // Alternatively, we can implement them by pairing a normal
 1394   // load/store with a memory barrier. For a volatile read we need
 1395   //
 1396   //   ldr<x>
 1397   //   dmb ishld
 1398   //
 1399   // for a volatile write
 1400   //
 1401   //   dmb ish
 1402   //   str<x>
 1403   //   dmb ish
 1404   //
 1405   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1406   // sequences. These are normally translated to an instruction
 1407   // sequence like the following
 1408   //
 1409   //   dmb      ish
 1410   // retry:
 1411   //   ldxr<x>   rval raddr
 1412   //   cmp       rval rold
 1413   //   b.ne done
 1414   //   stlxr<x>  rval, rnew, rold
 1415   //   cbnz      rval retry
 1416   // done:
 1417   //   cset      r0, eq
 1418   //   dmb ishld
 1419   //
 1420   // Note that the exclusive store is already using an stlxr
 1421   // instruction. That is required to ensure visibility to other
 1422   // threads of the exclusive write (assuming it succeeds) before that
 1423   // of any subsequent writes.
 1424   //
 1425   // The following instruction sequence is an improvement on the above
 1426   //
 1427   // retry:
 1428   //   ldaxr<x>  rval raddr
 1429   //   cmp       rval rold
 1430   //   b.ne done
 1431   //   stlxr<x>  rval, rnew, rold
 1432   //   cbnz      rval retry
 1433   // done:
 1434   //   cset      r0, eq
 1435   //
 1436   // We don't need the leading dmb ish since the stlxr guarantees
 1437   // visibility of prior writes in the case that the swap is
 1438   // successful. Crucially we don't have to worry about the case where
 1439   // the swap is not successful since no valid program should be
 1440   // relying on visibility of prior changes by the attempting thread
 1441   // in the case where the CAS fails.
 1442   //
 1443   // Similarly, we don't need the trailing dmb ishld if we substitute
 1444   // an ldaxr instruction since that will provide all the guarantees we
 1445   // require regarding observation of changes made by other threads
 1446   // before any change to the CAS address observed by the load.
 1447   //
 1448   // In order to generate the desired instruction sequence we need to
 1449   // be able to identify specific 'signature' ideal graph node
 1450   // sequences which i) occur as a translation of a volatile reads or
 1451   // writes or CAS operations and ii) do not occur through any other
 1452   // translation or graph transformation. We can then provide
 1453   // alternative aldc matching rules which translate these node
 1454   // sequences to the desired machine code sequences. Selection of the
 1455   // alternative rules can be implemented by predicates which identify
 1456   // the relevant node sequences.
 1457   //
 1458   // The ideal graph generator translates a volatile read to the node
 1459   // sequence
 1460   //
 1461   //   LoadX[mo_acquire]
 1462   //   MemBarAcquire
 1463   //
 1464   // As a special case when using the compressed oops optimization we
 1465   // may also see this variant
 1466   //
 1467   //   LoadN[mo_acquire]
 1468   //   DecodeN
 1469   //   MemBarAcquire
 1470   //
 1471   // A volatile write is translated to the node sequence
 1472   //
 1473   //   MemBarRelease
 1474   //   StoreX[mo_release] {CardMark}-optional
 1475   //   MemBarVolatile
 1476   //
 1477   // n.b. the above node patterns are generated with a strict
 1478   // 'signature' configuration of input and output dependencies (see
 1479   // the predicates below for exact details). The card mark may be as
 1480   // simple as a few extra nodes or, in a few GC configurations, may
 1481   // include more complex control flow between the leading and
 1482   // trailing memory barriers. However, whatever the card mark
 1483   // configuration these signatures are unique to translated volatile
 1484   // reads/stores -- they will not appear as a result of any other
 1485   // bytecode translation or inlining nor as a consequence of
 1486   // optimizing transforms.
 1487   //
 1488   // We also want to catch inlined unsafe volatile gets and puts and
 1489   // be able to implement them using either ldar<x>/stlr<x> or some
 1490   // combination of ldr<x>/stlr<x> and dmb instructions.
 1491   //
 1492   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1493   // normal volatile put node sequence containing an extra cpuorder
 1494   // membar
 1495   //
 1496   //   MemBarRelease
 1497   //   MemBarCPUOrder
 1498   //   StoreX[mo_release] {CardMark}-optional
 1499   //   MemBarCPUOrder
 1500   //   MemBarVolatile
 1501   //
 1502   // n.b. as an aside, a cpuorder membar is not itself subject to
 1503   // matching and translation by adlc rules.  However, the rule
 1504   // predicates need to detect its presence in order to correctly
 1505   // select the desired adlc rules.
 1506   //
 1507   // Inlined unsafe volatile gets manifest as a slightly different
 1508   // node sequence to a normal volatile get because of the
 1509   // introduction of some CPUOrder memory barriers to bracket the
 1510   // Load. However, but the same basic skeleton of a LoadX feeding a
 1511   // MemBarAcquire, possibly through an optional DecodeN, is still
 1512   // present
 1513   //
 1514   //   MemBarCPUOrder
 1515   //        ||       \\
 1516   //   MemBarCPUOrder LoadX[mo_acquire]
 1517   //        ||            |
 1518   //        ||       {DecodeN} optional
 1519   //        ||       /
 1520   //     MemBarAcquire
 1521   //
 1522   // In this case the acquire membar does not directly depend on the
 1523   // load. However, we can be sure that the load is generated from an
 1524   // inlined unsafe volatile get if we see it dependent on this unique
 1525   // sequence of membar nodes. Similarly, given an acquire membar we
 1526   // can know that it was added because of an inlined unsafe volatile
 1527   // get if it is fed and feeds a cpuorder membar and if its feed
 1528   // membar also feeds an acquiring load.
 1529   //
 1530   // Finally an inlined (Unsafe) CAS operation is translated to the
 1531   // following ideal graph
 1532   //
 1533   //   MemBarRelease
 1534   //   MemBarCPUOrder
 1535   //   CompareAndSwapX {CardMark}-optional
 1536   //   MemBarCPUOrder
 1537   //   MemBarAcquire
 1538   //
 1539   // So, where we can identify these volatile read and write
 1540   // signatures we can choose to plant either of the above two code
 1541   // sequences. For a volatile read we can simply plant a normal
 1542   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1543   // also choose to inhibit translation of the MemBarAcquire and
 1544   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1545   //
 1546   // When we recognise a volatile store signature we can choose to
 1547   // plant at a dmb ish as a translation for the MemBarRelease, a
 1548   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1549   // Alternatively, we can inhibit translation of the MemBarRelease
 1550   // and MemBarVolatile and instead plant a simple stlr<x>
 1551   // instruction.
 1552   //
 1553   // when we recognise a CAS signature we can choose to plant a dmb
 1554   // ish as a translation for the MemBarRelease, the conventional
 1555   // macro-instruction sequence for the CompareAndSwap node (which
 1556   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1557   // Alternatively, we can elide generation of the dmb instructions
 1558   // and plant the alternative CompareAndSwap macro-instruction
 1559   // sequence (which uses ldaxr<x>).
 1560   //
 1561   // Of course, the above only applies when we see these signature
 1562   // configurations. We still want to plant dmb instructions in any
 1563   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1564   // MemBarVolatile. For example, at the end of a constructor which
 1565   // writes final/volatile fields we will see a MemBarRelease
 1566   // instruction and this needs a 'dmb ish' lest we risk the
 1567   // constructed object being visible without making the
 1568   // final/volatile field writes visible.
 1569   //
 1570   // n.b. the translation rules below which rely on detection of the
 1571   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1572   // If we see anything other than the signature configurations we
 1573   // always just translate the loads and stores to ldr<x> and str<x>
 1574   // and translate acquire, release and volatile membars to the
 1575   // relevant dmb instructions.
 1576   //
 1577 
 1578   // is_CAS(int opcode, bool maybe_volatile)
 1579   //
 1580   // return true if opcode is one of the possible CompareAndSwapX
 1581   // values otherwise false.
 1582 
 1583   bool is_CAS(int opcode, bool maybe_volatile)
 1584   {
 1585     switch(opcode) {
 1586       // We handle these
 1587     case Op_CompareAndSwapI:
 1588     case Op_CompareAndSwapL:
 1589     case Op_CompareAndSwapP:
 1590     case Op_CompareAndSwapN:
 1591     case Op_ShenandoahCompareAndSwapP:
 1592     case Op_ShenandoahCompareAndSwapN:
 1593     case Op_CompareAndSwapB:
 1594     case Op_CompareAndSwapS:
 1595     case Op_GetAndSetI:
 1596     case Op_GetAndSetL:
 1597     case Op_GetAndSetP:
 1598     case Op_GetAndSetN:
 1599     case Op_GetAndAddI:
 1600     case Op_GetAndAddL:
 1601       return true;
 1602     case Op_CompareAndExchangeI:
 1603     case Op_CompareAndExchangeN:
 1604     case Op_CompareAndExchangeB:
 1605     case Op_CompareAndExchangeS:
 1606     case Op_CompareAndExchangeL:
 1607     case Op_CompareAndExchangeP:
 1608     case Op_WeakCompareAndSwapB:
 1609     case Op_WeakCompareAndSwapS:
 1610     case Op_WeakCompareAndSwapI:
 1611     case Op_WeakCompareAndSwapL:
 1612     case Op_WeakCompareAndSwapP:
 1613     case Op_WeakCompareAndSwapN:
 1614     case Op_ShenandoahWeakCompareAndSwapP:
 1615     case Op_ShenandoahWeakCompareAndSwapN:
 1616     case Op_ShenandoahCompareAndExchangeP:
 1617     case Op_ShenandoahCompareAndExchangeN:
 1618       return maybe_volatile;
 1619     default:
 1620       return false;
 1621     }
 1622   }
 1623 
 1624   // helper to determine the maximum number of Phi nodes we may need to
 1625   // traverse when searching from a card mark membar for the merge mem
 1626   // feeding a trailing membar or vice versa
 1627 
 1628 // predicates controlling emit of ldr<x>/ldar<x>
 1629 
 1630 bool unnecessary_acquire(const Node *barrier)
 1631 {
 1632   assert(barrier->is_MemBar(), "expecting a membar");
 1633 
 1634   MemBarNode* mb = barrier->as_MemBar();
 1635 
 1636   if (mb->trailing_load()) {
 1637     return true;
 1638   }
 1639 
 1640   if (mb->trailing_load_store()) {
 1641     Node* load_store = mb->in(MemBarNode::Precedent);
 1642     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1643     return is_CAS(load_store->Opcode(), true);
 1644   }
 1645 
 1646   return false;
 1647 }
 1648 
 1649 bool needs_acquiring_load(const Node *n)
 1650 {
 1651   assert(n->is_Load(), "expecting a load");
 1652   LoadNode *ld = n->as_Load();
 1653   return ld->is_acquire();
 1654 }
 1655 
 1656 bool unnecessary_release(const Node *n)
 1657 {
 1658   assert((n->is_MemBar() &&
 1659           n->Opcode() == Op_MemBarRelease),
 1660          "expecting a release membar");
 1661 
 1662   MemBarNode *barrier = n->as_MemBar();
 1663   if (!barrier->leading()) {
 1664     return false;
 1665   } else {
 1666     Node* trailing = barrier->trailing_membar();
 1667     MemBarNode* trailing_mb = trailing->as_MemBar();
 1668     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1669     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1670 
 1671     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1672     if (mem->is_Store()) {
 1673       assert(mem->as_Store()->is_release(), "");
 1674       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1675       return true;
 1676     } else {
 1677       assert(mem->is_LoadStore(), "");
 1678       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1679       return is_CAS(mem->Opcode(), true);
 1680     }
 1681   }
 1682   return false;
 1683 }
 1684 
 1685 bool unnecessary_volatile(const Node *n)
 1686 {
 1687   // assert n->is_MemBar();
 1688   MemBarNode *mbvol = n->as_MemBar();
 1689 
 1690   bool release = mbvol->trailing_store();
 1691   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1692 #ifdef ASSERT
 1693   if (release) {
 1694     Node* leading = mbvol->leading_membar();
 1695     assert(leading->Opcode() == Op_MemBarRelease, "");
 1696     assert(leading->as_MemBar()->leading_store(), "");
 1697     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1698   }
 1699 #endif
 1700 
 1701   return release;
 1702 }
 1703 
 1704 // predicates controlling emit of str<x>/stlr<x>
 1705 
 1706 bool needs_releasing_store(const Node *n)
 1707 {
 1708   // assert n->is_Store();
 1709   StoreNode *st = n->as_Store();
 1710   return st->trailing_membar() != NULL;
 1711 }
 1712 
 1713 // predicate controlling translation of CAS
 1714 //
 1715 // returns true if CAS needs to use an acquiring load otherwise false
 1716 
 1717 bool needs_acquiring_load_exclusive(const Node *n)
 1718 {
 1719   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1720   LoadStoreNode* ldst = n->as_LoadStore();
 1721   if (is_CAS(n->Opcode(), false)) {
 1722     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1723   } else {
 1724     return ldst->trailing_membar() != NULL;
 1725   }
 1726 
 1727   // so we can just return true here
 1728   return true;
 1729 }
 1730 
 1731 // Assert that the given node is not a variable shift.
 1732 bool assert_not_var_shift(const Node* n) {
 1733   assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
 1734   return true;
 1735 }
 1736 
 1737 #define __ _masm.
 1738 
 1739 // advance declarations for helper functions to convert register
 1740 // indices to register objects
 1741 
 1742 // the ad file has to provide implementations of certain methods
 1743 // expected by the generic code
 1744 //
 1745 // REQUIRED FUNCTIONALITY
 1746 
 1747 //=============================================================================
 1748 
 1749 // !!!!! Special hack to get all types of calls to specify the byte offset
 1750 //       from the start of the call to the point where the return address
 1751 //       will point.
 1752 
 1753 int MachCallStaticJavaNode::ret_addr_offset()
 1754 {
 1755   // call should be a simple bl
 1756   int off = 4;
 1757   return off;
 1758 }
 1759 
 1760 int MachCallDynamicJavaNode::ret_addr_offset()
 1761 {
 1762   return 16; // movz, movk, movk, bl
 1763 }
 1764 
 1765 int MachCallRuntimeNode::ret_addr_offset() {
 1766   // for generated stubs the call will be
 1767   //   bl(addr)
 1768   // or with far branches
 1769   //   bl(trampoline_stub)
 1770   // for real runtime callouts it will be six instructions
 1771   // see aarch64_enc_java_to_runtime
 1772   //   adr(rscratch2, retaddr)
 1773   //   lea(rscratch1, RuntimeAddress(addr)
 1774   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1775   //   blr(rscratch1)
 1776   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1777   if (cb) {
 1778     return 1 * NativeInstruction::instruction_size;
 1779   } else {
 1780     return 6 * NativeInstruction::instruction_size;
 1781   }
 1782 }
 1783 
 1784 int MachCallNativeNode::ret_addr_offset() {
 1785   // This is implemented using aarch64_enc_java_to_runtime as above.
 1786   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1787   if (cb) {
 1788     return 1 * NativeInstruction::instruction_size;
 1789   } else {
 1790     return 6 * NativeInstruction::instruction_size;
 1791   }
 1792 }
 1793 
 1794 //=============================================================================
 1795 
 1796 #ifndef PRODUCT
 1797 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1798   st->print("BREAKPOINT");
 1799 }
 1800 #endif
 1801 
 1802 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1803   C2_MacroAssembler _masm(&cbuf);
 1804   __ brk(0);
 1805 }
 1806 
 1807 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1808   return MachNode::size(ra_);
 1809 }
 1810 
 1811 //=============================================================================
 1812 
 1813 #ifndef PRODUCT
 1814   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1815     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1816   }
 1817 #endif
 1818 
 1819   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1820     C2_MacroAssembler _masm(&cbuf);
 1821     for (int i = 0; i < _count; i++) {
 1822       __ nop();
 1823     }
 1824   }
 1825 
 1826   uint MachNopNode::size(PhaseRegAlloc*) const {
 1827     return _count * NativeInstruction::instruction_size;
 1828   }
 1829 
 1830 //=============================================================================
 1831 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1832 
 1833 int ConstantTable::calculate_table_base_offset() const {
 1834   return 0;  // absolute addressing, no offset
 1835 }
 1836 
 1837 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1838 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1839   ShouldNotReachHere();
 1840 }
 1841 
 1842 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1843   // Empty encoding
 1844 }
 1845 
 1846 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1847   return 0;
 1848 }
 1849 
 1850 #ifndef PRODUCT
 1851 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1852   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1853 }
 1854 #endif
 1855 
 1856 #ifndef PRODUCT
 1857 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1858   Compile* C = ra_->C;
 1859 
 1860   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1861 
 1862   if (C->output()->need_stack_bang(framesize))
 1863     st->print("# stack bang size=%d\n\t", framesize);
 1864 
 1865   if (VM_Version::use_rop_protection()) {
 1866     st->print("ldr zr, [lr]\n\t");
 1867     st->print("pacia  lr, rfp\n\t");
 1868   }
 1869   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1870     st->print("sub  sp, sp, #%d\n\t", framesize);
 1871     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1872     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1873   } else {
 1874     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1875     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1876     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1877     st->print("sub  sp, sp, rscratch1");
 1878   }
 1879   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1880     st->print("\n\t");
 1881     st->print("ldr  rscratch1, [guard]\n\t");
 1882     st->print("dmb ishld\n\t");
 1883     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1884     st->print("cmp  rscratch1, rscratch2\n\t");
 1885     st->print("b.eq skip");
 1886     st->print("\n\t");
 1887     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1888     st->print("b skip\n\t");
 1889     st->print("guard: int\n\t");
 1890     st->print("\n\t");
 1891     st->print("skip:\n\t");
 1892   }
 1893 }
 1894 #endif
 1895 
 1896 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1897   Compile* C = ra_->C;
 1898   C2_MacroAssembler _masm(&cbuf);
 1899 
 1900   // n.b. frame size includes space for return pc and rfp
 1901   const int framesize = C->output()->frame_size_in_bytes();
 1902 
 1903   // insert a nop at the start of the prolog so we can patch in a
 1904   // branch if we need to invalidate the method later
 1905   __ nop();
 1906 
 1907   if (C->clinit_barrier_on_entry()) {
 1908     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1909 
 1910     Label L_skip_barrier;
 1911 
 1912     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1913     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1914     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1915     __ bind(L_skip_barrier);
 1916   }
 1917 
 1918   if (C->max_vector_size() > 0) {
 1919     __ reinitialize_ptrue();
 1920   }
 1921 
 1922   int bangsize = C->output()->bang_size_in_bytes();
 1923   if (C->output()->need_stack_bang(bangsize))
 1924     __ generate_stack_overflow_check(bangsize);
 1925 
 1926   __ build_frame(framesize);
 1927 
 1928   if (C->stub_function() == NULL) {
 1929     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1930     bs->nmethod_entry_barrier(&_masm);
 1931   }
 1932 
 1933   if (VerifyStackAtCalls) {
 1934     Unimplemented();
 1935   }
 1936 
 1937   C->output()->set_frame_complete(cbuf.insts_size());
 1938 
 1939   if (C->has_mach_constant_base_node()) {
 1940     // NOTE: We set the table base offset here because users might be
 1941     // emitted before MachConstantBaseNode.
 1942     ConstantTable& constant_table = C->output()->constant_table();
 1943     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1944   }
 1945 }
 1946 
 1947 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1948 {
 1949   return MachNode::size(ra_); // too many variables; just compute it
 1950                               // the hard way
 1951 }
 1952 
 1953 int MachPrologNode::reloc() const
 1954 {
 1955   return 0;
 1956 }
 1957 
 1958 //=============================================================================
 1959 
 1960 #ifndef PRODUCT
 1961 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1962   Compile* C = ra_->C;
 1963   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1964 
 1965   st->print("# pop frame %d\n\t",framesize);
 1966 
 1967   if (framesize == 0) {
 1968     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1969   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1970     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1971     st->print("add  sp, sp, #%d\n\t", framesize);
 1972   } else {
 1973     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1974     st->print("add  sp, sp, rscratch1\n\t");
 1975     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1976   }
 1977   if (VM_Version::use_rop_protection()) {
 1978     st->print("autia lr, rfp\n\t");
 1979     st->print("ldr zr, [lr]\n\t");
 1980   }
 1981 
 1982   if (do_polling() && C->is_method_compilation()) {
 1983     st->print("# test polling word\n\t");
 1984     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1985     st->print("cmp  sp, rscratch1\n\t");
 1986     st->print("bhi #slow_path");
 1987   }
 1988 }
 1989 #endif
 1990 
 1991 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1992   Compile* C = ra_->C;
 1993   C2_MacroAssembler _masm(&cbuf);
 1994   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1995 
 1996   __ remove_frame(framesize);
 1997 
 1998   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1999     __ reserved_stack_check();
 2000   }
 2001 
 2002   if (do_polling() && C->is_method_compilation()) {
 2003     Label dummy_label;
 2004     Label* code_stub = &dummy_label;
 2005     if (!C->output()->in_scratch_emit_size()) {
 2006       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 2007     }
 2008     __ relocate(relocInfo::poll_return_type);
 2009     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 2010   }
 2011 }
 2012 
 2013 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 2014   // Variable size. Determine dynamically.
 2015   return MachNode::size(ra_);
 2016 }
 2017 
 2018 int MachEpilogNode::reloc() const {
 2019   // Return number of relocatable values contained in this instruction.
 2020   return 1; // 1 for polling page.
 2021 }
 2022 
 2023 const Pipeline * MachEpilogNode::pipeline() const {
 2024   return MachNode::pipeline_class();
 2025 }
 2026 
 2027 //=============================================================================
 2028 
 2029 // Figure out which register class each belongs in: rc_int, rc_float or
 2030 // rc_stack.
 2031 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2032 
 2033 static enum RC rc_class(OptoReg::Name reg) {
 2034 
 2035   if (reg == OptoReg::Bad) {
 2036     return rc_bad;
 2037   }
 2038 
 2039   // we have 32 int registers * 2 halves
 2040   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2041 
 2042   if (reg < slots_of_int_registers) {
 2043     return rc_int;
 2044   }
 2045 
 2046   // we have 32 float register * 8 halves
 2047   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2048   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2049     return rc_float;
 2050   }
 2051 
 2052   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2053   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2054     return rc_predicate;
 2055   }
 2056 
 2057   // Between predicate regs & stack is the flags.
 2058   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2059 
 2060   return rc_stack;
 2061 }
 2062 
 2063 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2064   Compile* C = ra_->C;
 2065 
 2066   // Get registers to move.
 2067   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2068   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2069   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2070   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2071 
 2072   enum RC src_hi_rc = rc_class(src_hi);
 2073   enum RC src_lo_rc = rc_class(src_lo);
 2074   enum RC dst_hi_rc = rc_class(dst_hi);
 2075   enum RC dst_lo_rc = rc_class(dst_lo);
 2076 
 2077   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2078 
 2079   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 2080     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2081            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2082            "expected aligned-adjacent pairs");
 2083   }
 2084 
 2085   if (src_lo == dst_lo && src_hi == dst_hi) {
 2086     return 0;            // Self copy, no move.
 2087   }
 2088 
 2089   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2090               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2091   int src_offset = ra_->reg2offset(src_lo);
 2092   int dst_offset = ra_->reg2offset(dst_lo);
 2093 
 2094   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2095     uint ireg = ideal_reg();
 2096     if (ireg == Op_VecA && cbuf) {
 2097       C2_MacroAssembler _masm(cbuf);
 2098       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2099       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2100         // stack->stack
 2101         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2102                                                 sve_vector_reg_size_in_bytes);
 2103       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2104         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2105                             sve_vector_reg_size_in_bytes);
 2106       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2107         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2108                               sve_vector_reg_size_in_bytes);
 2109       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2110         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2111                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2112                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2113       } else {
 2114         ShouldNotReachHere();
 2115       }
 2116     } else if (cbuf) {
 2117       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2118       C2_MacroAssembler _masm(cbuf);
 2119       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2120       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2121         // stack->stack
 2122         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2123         if (ireg == Op_VecD) {
 2124           __ unspill(rscratch1, true, src_offset);
 2125           __ spill(rscratch1, true, dst_offset);
 2126         } else {
 2127           __ spill_copy128(src_offset, dst_offset);
 2128         }
 2129       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2130         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2131                ireg == Op_VecD ? __ T8B : __ T16B,
 2132                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2133       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2134         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2135                  ireg == Op_VecD ? __ D : __ Q,
 2136                  ra_->reg2offset(dst_lo));
 2137       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2138         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2139                    ireg == Op_VecD ? __ D : __ Q,
 2140                    ra_->reg2offset(src_lo));
 2141       } else {
 2142         ShouldNotReachHere();
 2143       }
 2144     }
 2145   } else if (cbuf) {
 2146     C2_MacroAssembler _masm(cbuf);
 2147     switch (src_lo_rc) {
 2148     case rc_int:
 2149       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2150         if (is64) {
 2151             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2152                    as_Register(Matcher::_regEncode[src_lo]));
 2153         } else {
 2154             C2_MacroAssembler _masm(cbuf);
 2155             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2156                     as_Register(Matcher::_regEncode[src_lo]));
 2157         }
 2158       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2159         if (is64) {
 2160             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2161                      as_Register(Matcher::_regEncode[src_lo]));
 2162         } else {
 2163             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2164                      as_Register(Matcher::_regEncode[src_lo]));
 2165         }
 2166       } else {                    // gpr --> stack spill
 2167         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2168         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2169       }
 2170       break;
 2171     case rc_float:
 2172       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2173         if (is64) {
 2174             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2175                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2176         } else {
 2177             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2178                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2179         }
 2180       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2181         if (is64) {
 2182             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2183                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2184         } else {
 2185             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2186                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2187         }
 2188       } else {                    // fpr --> stack spill
 2189         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2190         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2191                  is64 ? __ D : __ S, dst_offset);
 2192       }
 2193       break;
 2194     case rc_stack:
 2195       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2196         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2197       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2198         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2199                    is64 ? __ D : __ S, src_offset);
 2200       } else if (dst_lo_rc == rc_predicate) {
 2201         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2202                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2203       } else {                    // stack --> stack copy
 2204         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2205         if (ideal_reg() == Op_RegVectMask) {
 2206           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2207                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2208         } else {
 2209           __ unspill(rscratch1, is64, src_offset);
 2210           __ spill(rscratch1, is64, dst_offset);
 2211         }
 2212       }
 2213       break;
 2214     case rc_predicate:
 2215       if (dst_lo_rc == rc_predicate) {
 2216         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2217       } else if (dst_lo_rc == rc_stack) {
 2218         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2219                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2220       } else {
 2221         assert(false, "bad src and dst rc_class combination.");
 2222         ShouldNotReachHere();
 2223       }
 2224       break;
 2225     default:
 2226       assert(false, "bad rc_class for spill");
 2227       ShouldNotReachHere();
 2228     }
 2229   }
 2230 
 2231   if (st) {
 2232     st->print("spill ");
 2233     if (src_lo_rc == rc_stack) {
 2234       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2235     } else {
 2236       st->print("%s -> ", Matcher::regName[src_lo]);
 2237     }
 2238     if (dst_lo_rc == rc_stack) {
 2239       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2240     } else {
 2241       st->print("%s", Matcher::regName[dst_lo]);
 2242     }
 2243     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2244       int vsize = 0;
 2245       switch (ideal_reg()) {
 2246       case Op_VecD:
 2247         vsize = 64;
 2248         break;
 2249       case Op_VecX:
 2250         vsize = 128;
 2251         break;
 2252       case Op_VecA:
 2253         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2254         break;
 2255       default:
 2256         assert(false, "bad register type for spill");
 2257         ShouldNotReachHere();
 2258       }
 2259       st->print("\t# vector spill size = %d", vsize);
 2260     } else if (ideal_reg() == Op_RegVectMask) {
 2261       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2262       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2263       st->print("\t# predicate spill size = %d", vsize);
 2264     } else {
 2265       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2266     }
 2267   }
 2268 
 2269   return 0;
 2270 
 2271 }
 2272 
 2273 #ifndef PRODUCT
 2274 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2275   if (!ra_)
 2276     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2277   else
 2278     implementation(NULL, ra_, false, st);
 2279 }
 2280 #endif
 2281 
 2282 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2283   implementation(&cbuf, ra_, false, NULL);
 2284 }
 2285 
 2286 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2287   return MachNode::size(ra_);
 2288 }
 2289 
 2290 //=============================================================================
 2291 
 2292 #ifndef PRODUCT
 2293 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2294   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2295   int reg = ra_->get_reg_first(this);
 2296   st->print("add %s, rsp, #%d]\t# box lock",
 2297             Matcher::regName[reg], offset);
 2298 }
 2299 #endif
 2300 
 2301 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2302   C2_MacroAssembler _masm(&cbuf);
 2303 
 2304   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2305   int reg    = ra_->get_encode(this);
 2306 
 2307   // This add will handle any 24-bit signed offset. 24 bits allows an
 2308   // 8 megabyte stack frame.
 2309   __ add(as_Register(reg), sp, offset);
 2310 }
 2311 
 2312 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2313   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2314   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2315 
 2316   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2317     return NativeInstruction::instruction_size;
 2318   } else {
 2319     return 2 * NativeInstruction::instruction_size;
 2320   }
 2321 }
 2322 
 2323 //=============================================================================
 2324 
 2325 #ifndef PRODUCT
 2326 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2327 {
 2328   st->print_cr("# MachUEPNode");
 2329   if (UseCompressedClassPointers) {
 2330     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2331     if (CompressedKlassPointers::shift() != 0) {
 2332       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2333     }
 2334   } else {
 2335    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2336   }
 2337   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2338   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2339 }
 2340 #endif
 2341 
 2342 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2343 {
 2344   // This is the unverified entry point.
 2345   C2_MacroAssembler _masm(&cbuf);
 2346 
 2347   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2348   Label skip;
 2349   // TODO
 2350   // can we avoid this skip and still use a reloc?
 2351   __ br(Assembler::EQ, skip);
 2352   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2353   __ bind(skip);
 2354 }
 2355 
 2356 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2357 {
 2358   return MachNode::size(ra_);
 2359 }
 2360 
 2361 // REQUIRED EMIT CODE
 2362 
 2363 //=============================================================================
 2364 
 2365 // Emit exception handler code.
 2366 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2367 {
 2368   // mov rscratch1 #exception_blob_entry_point
 2369   // br rscratch1
 2370   // Note that the code buffer's insts_mark is always relative to insts.
 2371   // That's why we must use the macroassembler to generate a handler.
 2372   C2_MacroAssembler _masm(&cbuf);
 2373   address base = __ start_a_stub(size_exception_handler());
 2374   if (base == NULL) {
 2375     ciEnv::current()->record_failure("CodeCache is full");
 2376     return 0;  // CodeBuffer::expand failed
 2377   }
 2378   int offset = __ offset();
 2379   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2380   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2381   __ end_a_stub();
 2382   return offset;
 2383 }
 2384 
 2385 // Emit deopt handler code.
 2386 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2387 {
 2388   // Note that the code buffer's insts_mark is always relative to insts.
 2389   // That's why we must use the macroassembler to generate a handler.
 2390   C2_MacroAssembler _masm(&cbuf);
 2391   address base = __ start_a_stub(size_deopt_handler());
 2392   if (base == NULL) {
 2393     ciEnv::current()->record_failure("CodeCache is full");
 2394     return 0;  // CodeBuffer::expand failed
 2395   }
 2396   int offset = __ offset();
 2397 
 2398   __ adr(lr, __ pc());
 2399   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2400 
 2401   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2402   __ end_a_stub();
 2403   return offset;
 2404 }
 2405 
 2406 // REQUIRED MATCHER CODE
 2407 
 2408 //=============================================================================
 2409 
 2410 const bool Matcher::match_rule_supported(int opcode) {
 2411   if (!has_match_rule(opcode))
 2412     return false;
 2413 
 2414   bool ret_value = true;
 2415   switch (opcode) {
 2416     case Op_OnSpinWait:
 2417       return VM_Version::supports_on_spin_wait();
 2418     case Op_CacheWB:
 2419     case Op_CacheWBPreSync:
 2420     case Op_CacheWBPostSync:
 2421       if (!VM_Version::supports_data_cache_line_flush()) {
 2422         ret_value = false;
 2423       }
 2424       break;
 2425     case Op_LoadVectorMasked:
 2426     case Op_StoreVectorMasked:
 2427     case Op_LoadVectorGatherMasked:
 2428     case Op_StoreVectorScatterMasked:
 2429     case Op_MaskAll:
 2430     case Op_AndVMask:
 2431     case Op_OrVMask:
 2432     case Op_XorVMask:
 2433       if (UseSVE == 0) {
 2434         ret_value = false;
 2435       }
 2436       break;
 2437   }
 2438 
 2439   return ret_value; // Per default match rules are supported.
 2440 }
 2441 
 2442 // Identify extra cases that we might want to provide match rules for vector nodes and
 2443 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2444 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2445   if (!match_rule_supported(opcode)) {
 2446     return false;
 2447   }
 2448   int bit_size = vlen * type2aelembytes(bt) * 8;
 2449   if (UseSVE == 0 && bit_size > 128) {
 2450     return false;
 2451   }
 2452   if (UseSVE > 0) {
 2453     return op_sve_supported(opcode, vlen, bt);
 2454   } else { // NEON
 2455     // Special cases
 2456     switch (opcode) {
 2457     case Op_VectorMaskCmp:
 2458       if (vlen < 2 || bit_size < 64) {
 2459         return false;
 2460       }
 2461       break;
 2462     case Op_MulAddVS2VI:
 2463       if (bit_size < 128) {
 2464         return false;
 2465       }
 2466       break;
 2467     case Op_MulVL:
 2468       return false;
 2469     case Op_VectorLoadShuffle:
 2470     case Op_VectorRearrange:
 2471       if (vlen < 4) {
 2472         return false;
 2473       }
 2474       break;
 2475     case Op_LoadVectorGather:
 2476     case Op_StoreVectorScatter:
 2477       return false;
 2478     default:
 2479       break;
 2480     }
 2481   }
 2482   return vector_size_supported(bt, vlen);
 2483 }
 2484 
 2485 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
 2486   // Only SVE supports masked operations.
 2487   if (UseSVE == 0) {
 2488     return false;
 2489   }
 2490   return match_rule_supported(opcode) &&
 2491          masked_op_sve_supported(opcode, vlen, bt);
 2492 }
 2493 
 2494 const RegMask* Matcher::predicate_reg_mask(void) {
 2495   return &_PR_REG_mask;
 2496 }
 2497 
 2498 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2499   return new TypeVectMask(elemTy, length);
 2500 }
 2501 
 2502 // Vector calling convention not yet implemented.
 2503 const bool Matcher::supports_vector_calling_convention(void) {
 2504   return false;
 2505 }
 2506 
 2507 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2508   Unimplemented();
 2509   return OptoRegPair(0, 0);
 2510 }
 2511 
 2512 // Is this branch offset short enough that a short branch can be used?
 2513 //
 2514 // NOTE: If the platform does not provide any short branch variants, then
 2515 //       this method should return false for offset 0.
 2516 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2517   // The passed offset is relative to address of the branch.
 2518 
 2519   return (-32768 <= offset && offset < 32768);
 2520 }
 2521 
 2522 // Vector width in bytes.
 2523 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2524   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2525   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2526   // Minimum 2 values in vector
 2527   if (size < 2*type2aelembytes(bt)) size = 0;
 2528   // But never < 4
 2529   if (size < 4) size = 0;
 2530   return size;
 2531 }
 2532 
 2533 // Limits on vector size (number of elements) loaded into vector.
 2534 const int Matcher::max_vector_size(const BasicType bt) {
 2535   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2536 }
 2537 
 2538 const int Matcher::min_vector_size(const BasicType bt) {
 2539   int max_size = max_vector_size(bt);
 2540   // Limit the min vector size to 8 bytes.
 2541   int size = 8 / type2aelembytes(bt);
 2542   if (bt == T_BYTE) {
 2543     // To support vector api shuffle/rearrange.
 2544     size = 4;
 2545   } else if (bt == T_BOOLEAN) {
 2546     // To support vector api load/store mask.
 2547     size = 2;
 2548   }
 2549   if (size < 2) size = 2;
 2550   return MIN2(size, max_size);
 2551 }
 2552 
 2553 // Actual max scalable vector register length.
 2554 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2555   return Matcher::max_vector_size(bt);
 2556 }
 2557 
 2558 // Vector ideal reg.
 2559 const uint Matcher::vector_ideal_reg(int len) {
 2560   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2561     return Op_VecA;
 2562   }
 2563   switch(len) {
 2564     // For 16-bit/32-bit mask vector, reuse VecD.
 2565     case  2:
 2566     case  4:
 2567     case  8: return Op_VecD;
 2568     case 16: return Op_VecX;
 2569   }
 2570   ShouldNotReachHere();
 2571   return 0;
 2572 }
 2573 
 2574 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2575   ShouldNotReachHere(); // generic vector operands not supported
 2576   return NULL;
 2577 }
 2578 
 2579 bool Matcher::is_reg2reg_move(MachNode* m) {
 2580   ShouldNotReachHere();  // generic vector operands not supported
 2581   return false;
 2582 }
 2583 
 2584 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2585   ShouldNotReachHere();  // generic vector operands not supported
 2586   return false;
 2587 }
 2588 
 2589 // Return whether or not this register is ever used as an argument.
 2590 // This function is used on startup to build the trampoline stubs in
 2591 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2592 // call in the trampoline, and arguments in those registers not be
 2593 // available to the callee.
 2594 bool Matcher::can_be_java_arg(int reg)
 2595 {
 2596   return
 2597     reg ==  R0_num || reg == R0_H_num ||
 2598     reg ==  R1_num || reg == R1_H_num ||
 2599     reg ==  R2_num || reg == R2_H_num ||
 2600     reg ==  R3_num || reg == R3_H_num ||
 2601     reg ==  R4_num || reg == R4_H_num ||
 2602     reg ==  R5_num || reg == R5_H_num ||
 2603     reg ==  R6_num || reg == R6_H_num ||
 2604     reg ==  R7_num || reg == R7_H_num ||
 2605     reg ==  V0_num || reg == V0_H_num ||
 2606     reg ==  V1_num || reg == V1_H_num ||
 2607     reg ==  V2_num || reg == V2_H_num ||
 2608     reg ==  V3_num || reg == V3_H_num ||
 2609     reg ==  V4_num || reg == V4_H_num ||
 2610     reg ==  V5_num || reg == V5_H_num ||
 2611     reg ==  V6_num || reg == V6_H_num ||
 2612     reg ==  V7_num || reg == V7_H_num;
 2613 }
 2614 
 2615 bool Matcher::is_spillable_arg(int reg)
 2616 {
 2617   return can_be_java_arg(reg);
 2618 }
 2619 
 2620 uint Matcher::int_pressure_limit()
 2621 {
 2622   // JDK-8183543: When taking the number of available registers as int
 2623   // register pressure threshold, the jtreg test:
 2624   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2625   // failed due to C2 compilation failure with
 2626   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2627   //
 2628   // A derived pointer is live at CallNode and then is flagged by RA
 2629   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2630   // derived pointers and lastly fail to spill after reaching maximum
 2631   // number of iterations. Lowering the default pressure threshold to
 2632   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2633   // a high register pressure area of the code so that split_DEF can
 2634   // generate DefinitionSpillCopy for the derived pointer.
 2635   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2636   if (!PreserveFramePointer) {
 2637     // When PreserveFramePointer is off, frame pointer is allocatable,
 2638     // but different from other SOC registers, it is excluded from
 2639     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2640     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2641     // See check_pressure_at_fatproj().
 2642     default_int_pressure_threshold--;
 2643   }
 2644   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2645 }
 2646 
 2647 uint Matcher::float_pressure_limit()
 2648 {
 2649   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2650   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2651 }
 2652 
 2653 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2654   return false;
 2655 }
 2656 
 2657 RegMask Matcher::divI_proj_mask() {
 2658   ShouldNotReachHere();
 2659   return RegMask();
 2660 }
 2661 
 2662 // Register for MODI projection of divmodI.
 2663 RegMask Matcher::modI_proj_mask() {
 2664   ShouldNotReachHere();
 2665   return RegMask();
 2666 }
 2667 
 2668 // Register for DIVL projection of divmodL.
 2669 RegMask Matcher::divL_proj_mask() {
 2670   ShouldNotReachHere();
 2671   return RegMask();
 2672 }
 2673 
 2674 // Register for MODL projection of divmodL.
 2675 RegMask Matcher::modL_proj_mask() {
 2676   ShouldNotReachHere();
 2677   return RegMask();
 2678 }
 2679 
 2680 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2681   return FP_REG_mask();
 2682 }
 2683 
 2684 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2685   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2686     Node* u = addp->fast_out(i);
 2687     if (u->is_LoadStore()) {
 2688       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2689       // instructions) only take register indirect as an operand, so
 2690       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2691       // must fail.
 2692       return false;
 2693     }
 2694     if (u->is_Mem()) {
 2695       int opsize = u->as_Mem()->memory_size();
 2696       assert(opsize > 0, "unexpected memory operand size");
 2697       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2698         return false;
 2699       }
 2700     }
 2701   }
 2702   return true;
 2703 }
 2704 
 2705 bool can_combine_with_imm(Node* binary_node, Node* replicate_node) {
 2706   if (UseSVE == 0 || !VectorNode::is_invariant_vector(replicate_node)){
 2707     return false;
 2708   }
 2709   Node* imm_node = replicate_node->in(1);
 2710   if (!imm_node->is_Con()) {
 2711     return false;
 2712   }
 2713 
 2714   const Type* t = imm_node->bottom_type();
 2715   if (!(t->isa_int() || t->isa_long())) {
 2716     return false;
 2717   }
 2718 
 2719   switch (binary_node->Opcode()) {
 2720   case Op_AndV:
 2721   case Op_OrV:
 2722   case Op_XorV: {
 2723     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(binary_node));
 2724     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2725     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2726   }
 2727   case Op_AddVB:
 2728     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2729   case Op_AddVS:
 2730   case Op_AddVI:
 2731     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2732   case Op_AddVL:
 2733     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2734   default:
 2735     return false;
 2736   }
 2737 }
 2738 
 2739 bool is_vector_arith_imm_pattern(Node* n, Node* m) {
 2740   if (n != NULL && m != NULL) {
 2741     return can_combine_with_imm(n, m);
 2742   }
 2743   return false;
 2744 }
 2745 
 2746 // Should the matcher clone input 'm' of node 'n'?
 2747 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2748   // ShiftV src (ShiftCntV con)
 2749   // Binary src (Replicate con)
 2750   if (is_vshift_con_pattern(n, m) || is_vector_arith_imm_pattern(n, m)) {
 2751     mstack.push(m, Visit);
 2752     return true;
 2753   }
 2754 
 2755   return false;
 2756 }
 2757 
 2758 // Should the Matcher clone shifts on addressing modes, expecting them
 2759 // to be subsumed into complex addressing expressions or compute them
 2760 // into registers?
 2761 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2762   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2763     return true;
 2764   }
 2765 
 2766   Node *off = m->in(AddPNode::Offset);
 2767   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2768       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2769       // Are there other uses besides address expressions?
 2770       !is_visited(off)) {
 2771     address_visited.set(off->_idx); // Flag as address_visited
 2772     mstack.push(off->in(2), Visit);
 2773     Node *conv = off->in(1);
 2774     if (conv->Opcode() == Op_ConvI2L &&
 2775         // Are there other uses besides address expressions?
 2776         !is_visited(conv)) {
 2777       address_visited.set(conv->_idx); // Flag as address_visited
 2778       mstack.push(conv->in(1), Pre_Visit);
 2779     } else {
 2780       mstack.push(conv, Pre_Visit);
 2781     }
 2782     address_visited.test_set(m->_idx); // Flag as address_visited
 2783     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2784     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2785     return true;
 2786   } else if (off->Opcode() == Op_ConvI2L &&
 2787              // Are there other uses besides address expressions?
 2788              !is_visited(off)) {
 2789     address_visited.test_set(m->_idx); // Flag as address_visited
 2790     address_visited.set(off->_idx); // Flag as address_visited
 2791     mstack.push(off->in(1), Pre_Visit);
 2792     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2793     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2794     return true;
 2795   }
 2796   return false;
 2797 }
 2798 
 2799 bool Parse::do_one_bytecode_targeted() {
 2800   return false;
 2801 }
 2802 
 2803 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2804   C2_MacroAssembler _masm(&cbuf);                                       \
 2805   {                                                                     \
 2806     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2807     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2808     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2809     __ INSN(REG, as_Register(BASE));                                    \
 2810   }
 2811 
 2812 
 2813 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2814   {
 2815     Address::extend scale;
 2816 
 2817     // Hooboy, this is fugly.  We need a way to communicate to the
 2818     // encoder that the index needs to be sign extended, so we have to
 2819     // enumerate all the cases.
 2820     switch (opcode) {
 2821     case INDINDEXSCALEDI2L:
 2822     case INDINDEXSCALEDI2LN:
 2823     case INDINDEXI2L:
 2824     case INDINDEXI2LN:
 2825       scale = Address::sxtw(size);
 2826       break;
 2827     default:
 2828       scale = Address::lsl(size);
 2829     }
 2830 
 2831     if (index == -1) {
 2832       return Address(base, disp);
 2833     } else {
 2834       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2835       return Address(base, as_Register(index), scale);
 2836     }
 2837   }
 2838 
 2839 
 2840 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2841 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2842 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2843 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2844                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2845 
 2846   // Used for all non-volatile memory accesses.  The use of
 2847   // $mem->opcode() to discover whether this pattern uses sign-extended
 2848   // offsets is something of a kludge.
 2849   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2850                         Register reg, int opcode,
 2851                         Register base, int index, int scale, int disp,
 2852                         int size_in_memory)
 2853   {
 2854     Address addr = mem2address(opcode, base, index, scale, disp);
 2855     if (addr.getMode() == Address::base_plus_offset) {
 2856       /* If we get an out-of-range offset it is a bug in the compiler,
 2857          so we assert here. */
 2858       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2859              "c2 compiler bug");
 2860       /* Fix up any out-of-range offsets. */
 2861       assert_different_registers(rscratch1, base);
 2862       assert_different_registers(rscratch1, reg);
 2863       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2864     }
 2865     (masm.*insn)(reg, addr);
 2866   }
 2867 
 2868   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2869                         FloatRegister reg, int opcode,
 2870                         Register base, int index, int size, int disp,
 2871                         int size_in_memory)
 2872   {
 2873     Address::extend scale;
 2874 
 2875     switch (opcode) {
 2876     case INDINDEXSCALEDI2L:
 2877     case INDINDEXSCALEDI2LN:
 2878       scale = Address::sxtw(size);
 2879       break;
 2880     default:
 2881       scale = Address::lsl(size);
 2882     }
 2883 
 2884     if (index == -1) {
 2885       /* If we get an out-of-range offset it is a bug in the compiler,
 2886          so we assert here. */
 2887       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2888       /* Fix up any out-of-range offsets. */
 2889       assert_different_registers(rscratch1, base);
 2890       Address addr = Address(base, disp);
 2891       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2892       (masm.*insn)(reg, addr);
 2893     } else {
 2894       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2895       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2896     }
 2897   }
 2898 
 2899   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2900                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2901                         int opcode, Register base, int index, int size, int disp)
 2902   {
 2903     if (index == -1) {
 2904       (masm.*insn)(reg, T, Address(base, disp));
 2905     } else {
 2906       assert(disp == 0, "unsupported address mode");
 2907       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2908     }
 2909   }
 2910 
 2911 %}
 2912 
 2913 
 2914 
 2915 //----------ENCODING BLOCK-----------------------------------------------------
 2916 // This block specifies the encoding classes used by the compiler to
 2917 // output byte streams.  Encoding classes are parameterized macros
 2918 // used by Machine Instruction Nodes in order to generate the bit
 2919 // encoding of the instruction.  Operands specify their base encoding
 2920 // interface with the interface keyword.  There are currently
 2921 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2922 // COND_INTER.  REG_INTER causes an operand to generate a function
 2923 // which returns its register number when queried.  CONST_INTER causes
 2924 // an operand to generate a function which returns the value of the
 2925 // constant when queried.  MEMORY_INTER causes an operand to generate
 2926 // four functions which return the Base Register, the Index Register,
 2927 // the Scale Value, and the Offset Value of the operand when queried.
 2928 // COND_INTER causes an operand to generate six functions which return
 2929 // the encoding code (ie - encoding bits for the instruction)
 2930 // associated with each basic boolean condition for a conditional
 2931 // instruction.
 2932 //
 2933 // Instructions specify two basic values for encoding.  Again, a
 2934 // function is available to check if the constant displacement is an
 2935 // oop. They use the ins_encode keyword to specify their encoding
 2936 // classes (which must be a sequence of enc_class names, and their
 2937 // parameters, specified in the encoding block), and they use the
 2938 // opcode keyword to specify, in order, their primary, secondary, and
 2939 // tertiary opcode.  Only the opcode sections which a particular
 2940 // instruction needs for encoding need to be specified.
 2941 encode %{
 2942   // Build emit functions for each basic byte or larger field in the
 2943   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2944   // from C++ code in the enc_class source block.  Emit functions will
 2945   // live in the main source block for now.  In future, we can
 2946   // generalize this by adding a syntax that specifies the sizes of
 2947   // fields in an order, so that the adlc can build the emit functions
 2948   // automagically
 2949 
 2950   // catch all for unimplemented encodings
 2951   enc_class enc_unimplemented %{
 2952     C2_MacroAssembler _masm(&cbuf);
 2953     __ unimplemented("C2 catch all");
 2954   %}
 2955 
 2956   // BEGIN Non-volatile memory access
 2957 
 2958   // This encoding class is generated automatically from ad_encode.m4.
 2959   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2960   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2961     Register dst_reg = as_Register($dst$$reg);
 2962     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2963                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2964   %}
 2965 
 2966   // This encoding class is generated automatically from ad_encode.m4.
 2967   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2968   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2969     Register dst_reg = as_Register($dst$$reg);
 2970     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2971                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2972   %}
 2973 
 2974   // This encoding class is generated automatically from ad_encode.m4.
 2975   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2976   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2977     Register dst_reg = as_Register($dst$$reg);
 2978     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2979                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2980   %}
 2981 
 2982   // This encoding class is generated automatically from ad_encode.m4.
 2983   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2984   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2985     Register dst_reg = as_Register($dst$$reg);
 2986     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2987                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2988   %}
 2989 
 2990   // This encoding class is generated automatically from ad_encode.m4.
 2991   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2992   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2993     Register dst_reg = as_Register($dst$$reg);
 2994     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2995                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2996   %}
 2997 
 2998   // This encoding class is generated automatically from ad_encode.m4.
 2999   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3000   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 3001     Register dst_reg = as_Register($dst$$reg);
 3002     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 3003                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3004   %}
 3005 
 3006   // This encoding class is generated automatically from ad_encode.m4.
 3007   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3008   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 3009     Register dst_reg = as_Register($dst$$reg);
 3010     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3011                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3012   %}
 3013 
 3014   // This encoding class is generated automatically from ad_encode.m4.
 3015   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3016   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 3017     Register dst_reg = as_Register($dst$$reg);
 3018     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3019                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3020   %}
 3021 
 3022   // This encoding class is generated automatically from ad_encode.m4.
 3023   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3024   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 3025     Register dst_reg = as_Register($dst$$reg);
 3026     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3027                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3028   %}
 3029 
 3030   // This encoding class is generated automatically from ad_encode.m4.
 3031   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3032   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 3033     Register dst_reg = as_Register($dst$$reg);
 3034     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3035                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3036   %}
 3037 
 3038   // This encoding class is generated automatically from ad_encode.m4.
 3039   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3040   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 3041     Register dst_reg = as_Register($dst$$reg);
 3042     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 3043                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3044   %}
 3045 
 3046   // This encoding class is generated automatically from ad_encode.m4.
 3047   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3048   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 3049     Register dst_reg = as_Register($dst$$reg);
 3050     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 3051                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3052   %}
 3053 
 3054   // This encoding class is generated automatically from ad_encode.m4.
 3055   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3056   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3057     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3058     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3059                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3060   %}
 3061 
 3062   // This encoding class is generated automatically from ad_encode.m4.
 3063   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3064   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3065     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3066     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3067                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3068   %}
 3069 
 3070   // This encoding class is generated automatically from ad_encode.m4.
 3071   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3072   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3073     Register src_reg = as_Register($src$$reg);
 3074     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 3075                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3076   %}
 3077 
 3078   // This encoding class is generated automatically from ad_encode.m4.
 3079   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3080   enc_class aarch64_enc_strb0(memory1 mem) %{
 3081     C2_MacroAssembler _masm(&cbuf);
 3082     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3083                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3084   %}
 3085 
 3086   // This encoding class is generated automatically from ad_encode.m4.
 3087   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3088   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3089     Register src_reg = as_Register($src$$reg);
 3090     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3091                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3092   %}
 3093 
 3094   // This encoding class is generated automatically from ad_encode.m4.
 3095   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3096   enc_class aarch64_enc_strh0(memory2 mem) %{
 3097     C2_MacroAssembler _masm(&cbuf);
 3098     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3099                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3100   %}
 3101 
 3102   // This encoding class is generated automatically from ad_encode.m4.
 3103   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3104   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3105     Register src_reg = as_Register($src$$reg);
 3106     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3107                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3108   %}
 3109 
 3110   // This encoding class is generated automatically from ad_encode.m4.
 3111   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3112   enc_class aarch64_enc_strw0(memory4 mem) %{
 3113     C2_MacroAssembler _masm(&cbuf);
 3114     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3115                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3116   %}
 3117 
 3118   // This encoding class is generated automatically from ad_encode.m4.
 3119   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3120   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3121     Register src_reg = as_Register($src$$reg);
 3122     // we sometimes get asked to store the stack pointer into the
 3123     // current thread -- we cannot do that directly on AArch64
 3124     if (src_reg == r31_sp) {
 3125       C2_MacroAssembler _masm(&cbuf);
 3126       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3127       __ mov(rscratch2, sp);
 3128       src_reg = rscratch2;
 3129     }
 3130     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3131                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3132   %}
 3133 
 3134   // This encoding class is generated automatically from ad_encode.m4.
 3135   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3136   enc_class aarch64_enc_str0(memory8 mem) %{
 3137     C2_MacroAssembler _masm(&cbuf);
 3138     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3139                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3140   %}
 3141 
 3142   // This encoding class is generated automatically from ad_encode.m4.
 3143   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3144   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3145     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3146     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3147                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3148   %}
 3149 
 3150   // This encoding class is generated automatically from ad_encode.m4.
 3151   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3152   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3153     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3154     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3155                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3156   %}
 3157 
 3158   // This encoding class is generated automatically from ad_encode.m4.
 3159   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3160   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3161       C2_MacroAssembler _masm(&cbuf);
 3162       __ membar(Assembler::StoreStore);
 3163       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3164                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3165   %}
 3166 
 3167   // END Non-volatile memory access
 3168 
 3169   // Vector loads and stores
 3170   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3171     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3172     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3173        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3174   %}
 3175 
 3176   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3177     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3178     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3179        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3180   %}
 3181 
 3182   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3183     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3184     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3185        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3186   %}
 3187 
 3188   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3189     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3190     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3191        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3192   %}
 3193 
 3194   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3195     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3196     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3197        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3198   %}
 3199 
 3200   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3201     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3202     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3203        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3204   %}
 3205 
 3206   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3207     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3208     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3209        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3210   %}
 3211 
 3212   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3213     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3214     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3215        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3216   %}
 3217 
 3218   // volatile loads and stores
 3219 
 3220   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3221     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3222                  rscratch1, stlrb);
 3223   %}
 3224 
 3225   enc_class aarch64_enc_stlrb0(memory mem) %{
 3226     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3227                  rscratch1, stlrb);
 3228   %}
 3229 
 3230   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3231     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3232                  rscratch1, stlrh);
 3233   %}
 3234 
 3235   enc_class aarch64_enc_stlrh0(memory mem) %{
 3236     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3237                  rscratch1, stlrh);
 3238   %}
 3239 
 3240   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3241     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3242                  rscratch1, stlrw);
 3243   %}
 3244 
 3245   enc_class aarch64_enc_stlrw0(memory mem) %{
 3246     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3247                  rscratch1, stlrw);
 3248   %}
 3249 
 3250   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3251     Register dst_reg = as_Register($dst$$reg);
 3252     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3253              rscratch1, ldarb);
 3254     __ sxtbw(dst_reg, dst_reg);
 3255   %}
 3256 
 3257   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3258     Register dst_reg = as_Register($dst$$reg);
 3259     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3260              rscratch1, ldarb);
 3261     __ sxtb(dst_reg, dst_reg);
 3262   %}
 3263 
 3264   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3265     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3266              rscratch1, ldarb);
 3267   %}
 3268 
 3269   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3270     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3271              rscratch1, ldarb);
 3272   %}
 3273 
 3274   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3275     Register dst_reg = as_Register($dst$$reg);
 3276     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3277              rscratch1, ldarh);
 3278     __ sxthw(dst_reg, dst_reg);
 3279   %}
 3280 
 3281   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3282     Register dst_reg = as_Register($dst$$reg);
 3283     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3284              rscratch1, ldarh);
 3285     __ sxth(dst_reg, dst_reg);
 3286   %}
 3287 
 3288   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3289     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3290              rscratch1, ldarh);
 3291   %}
 3292 
 3293   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3294     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3295              rscratch1, ldarh);
 3296   %}
 3297 
 3298   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3299     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3300              rscratch1, ldarw);
 3301   %}
 3302 
 3303   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3304     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3305              rscratch1, ldarw);
 3306   %}
 3307 
 3308   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3309     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3310              rscratch1, ldar);
 3311   %}
 3312 
 3313   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3314     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3315              rscratch1, ldarw);
 3316     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3317   %}
 3318 
 3319   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3320     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3321              rscratch1, ldar);
 3322     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3323   %}
 3324 
 3325   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3326     Register src_reg = as_Register($src$$reg);
 3327     // we sometimes get asked to store the stack pointer into the
 3328     // current thread -- we cannot do that directly on AArch64
 3329     if (src_reg == r31_sp) {
 3330       C2_MacroAssembler _masm(&cbuf);
 3331       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3332       __ mov(rscratch2, sp);
 3333       src_reg = rscratch2;
 3334     }
 3335     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3336                  rscratch1, stlr);
 3337   %}
 3338 
 3339   enc_class aarch64_enc_stlr0(memory mem) %{
 3340     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3341                  rscratch1, stlr);
 3342   %}
 3343 
 3344   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3345     {
 3346       C2_MacroAssembler _masm(&cbuf);
 3347       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3348       __ fmovs(rscratch2, src_reg);
 3349     }
 3350     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3351                  rscratch1, stlrw);
 3352   %}
 3353 
 3354   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3355     {
 3356       C2_MacroAssembler _masm(&cbuf);
 3357       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3358       __ fmovd(rscratch2, src_reg);
 3359     }
 3360     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3361                  rscratch1, stlr);
 3362   %}
 3363 
 3364   // synchronized read/update encodings
 3365 
 3366   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3367     C2_MacroAssembler _masm(&cbuf);
 3368     Register dst_reg = as_Register($dst$$reg);
 3369     Register base = as_Register($mem$$base);
 3370     int index = $mem$$index;
 3371     int scale = $mem$$scale;
 3372     int disp = $mem$$disp;
 3373     if (index == -1) {
 3374        if (disp != 0) {
 3375         __ lea(rscratch1, Address(base, disp));
 3376         __ ldaxr(dst_reg, rscratch1);
 3377       } else {
 3378         // TODO
 3379         // should we ever get anything other than this case?
 3380         __ ldaxr(dst_reg, base);
 3381       }
 3382     } else {
 3383       Register index_reg = as_Register(index);
 3384       if (disp == 0) {
 3385         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3386         __ ldaxr(dst_reg, rscratch1);
 3387       } else {
 3388         __ lea(rscratch1, Address(base, disp));
 3389         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3390         __ ldaxr(dst_reg, rscratch1);
 3391       }
 3392     }
 3393   %}
 3394 
 3395   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3396     C2_MacroAssembler _masm(&cbuf);
 3397     Register src_reg = as_Register($src$$reg);
 3398     Register base = as_Register($mem$$base);
 3399     int index = $mem$$index;
 3400     int scale = $mem$$scale;
 3401     int disp = $mem$$disp;
 3402     if (index == -1) {
 3403        if (disp != 0) {
 3404         __ lea(rscratch2, Address(base, disp));
 3405         __ stlxr(rscratch1, src_reg, rscratch2);
 3406       } else {
 3407         // TODO
 3408         // should we ever get anything other than this case?
 3409         __ stlxr(rscratch1, src_reg, base);
 3410       }
 3411     } else {
 3412       Register index_reg = as_Register(index);
 3413       if (disp == 0) {
 3414         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3415         __ stlxr(rscratch1, src_reg, rscratch2);
 3416       } else {
 3417         __ lea(rscratch2, Address(base, disp));
 3418         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3419         __ stlxr(rscratch1, src_reg, rscratch2);
 3420       }
 3421     }
 3422     __ cmpw(rscratch1, zr);
 3423   %}
 3424 
 3425   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3426     C2_MacroAssembler _masm(&cbuf);
 3427     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3428     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3429                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3430                /*weak*/ false, noreg);
 3431   %}
 3432 
 3433   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3434     C2_MacroAssembler _masm(&cbuf);
 3435     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3436     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3437                Assembler::word, /*acquire*/ false, /*release*/ true,
 3438                /*weak*/ false, noreg);
 3439   %}
 3440 
 3441   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3442     C2_MacroAssembler _masm(&cbuf);
 3443     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3444     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3445                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3446                /*weak*/ false, noreg);
 3447   %}
 3448 
 3449   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3450     C2_MacroAssembler _masm(&cbuf);
 3451     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3452     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3453                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3454                /*weak*/ false, noreg);
 3455   %}
 3456 
 3457 
 3458   // The only difference between aarch64_enc_cmpxchg and
 3459   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3460   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3461   // lock.
 3462   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3463     C2_MacroAssembler _masm(&cbuf);
 3464     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3465     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3466                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3467                /*weak*/ false, noreg);
 3468   %}
 3469 
 3470   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3471     C2_MacroAssembler _masm(&cbuf);
 3472     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3473     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3474                Assembler::word, /*acquire*/ true, /*release*/ true,
 3475                /*weak*/ false, noreg);
 3476   %}
 3477 
 3478   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3481     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3482                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3483                /*weak*/ false, noreg);
 3484   %}
 3485 
 3486   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3487     C2_MacroAssembler _masm(&cbuf);
 3488     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3489     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3490                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3491                /*weak*/ false, noreg);
 3492   %}
 3493 
 3494   // auxiliary used for CompareAndSwapX to set result register
 3495   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3496     C2_MacroAssembler _masm(&cbuf);
 3497     Register res_reg = as_Register($res$$reg);
 3498     __ cset(res_reg, Assembler::EQ);
 3499   %}
 3500 
 3501   // prefetch encodings
 3502 
 3503   enc_class aarch64_enc_prefetchw(memory mem) %{
 3504     C2_MacroAssembler _masm(&cbuf);
 3505     Register base = as_Register($mem$$base);
 3506     int index = $mem$$index;
 3507     int scale = $mem$$scale;
 3508     int disp = $mem$$disp;
 3509     if (index == -1) {
 3510       __ prfm(Address(base, disp), PSTL1KEEP);
 3511     } else {
 3512       Register index_reg = as_Register(index);
 3513       if (disp == 0) {
 3514         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3515       } else {
 3516         __ lea(rscratch1, Address(base, disp));
 3517 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3518       }
 3519     }
 3520   %}
 3521 
 3522   /// mov envcodings
 3523 
 3524   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3525     C2_MacroAssembler _masm(&cbuf);
 3526     uint32_t con = (uint32_t)$src$$constant;
 3527     Register dst_reg = as_Register($dst$$reg);
 3528     if (con == 0) {
 3529       __ movw(dst_reg, zr);
 3530     } else {
 3531       __ movw(dst_reg, con);
 3532     }
 3533   %}
 3534 
 3535   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3536     C2_MacroAssembler _masm(&cbuf);
 3537     Register dst_reg = as_Register($dst$$reg);
 3538     uint64_t con = (uint64_t)$src$$constant;
 3539     if (con == 0) {
 3540       __ mov(dst_reg, zr);
 3541     } else {
 3542       __ mov(dst_reg, con);
 3543     }
 3544   %}
 3545 
 3546   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3547     C2_MacroAssembler _masm(&cbuf);
 3548     Register dst_reg = as_Register($dst$$reg);
 3549     address con = (address)$src$$constant;
 3550     if (con == NULL || con == (address)1) {
 3551       ShouldNotReachHere();
 3552     } else {
 3553       relocInfo::relocType rtype = $src->constant_reloc();
 3554       if (rtype == relocInfo::oop_type) {
 3555         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3556       } else if (rtype == relocInfo::metadata_type) {
 3557         __ mov_metadata(dst_reg, (Metadata*)con);
 3558       } else {
 3559         assert(rtype == relocInfo::none, "unexpected reloc type");
 3560         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3561           __ mov(dst_reg, con);
 3562         } else {
 3563           uint64_t offset;
 3564           __ adrp(dst_reg, con, offset);
 3565           __ add(dst_reg, dst_reg, offset);
 3566         }
 3567       }
 3568     }
 3569   %}
 3570 
 3571   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3572     C2_MacroAssembler _masm(&cbuf);
 3573     Register dst_reg = as_Register($dst$$reg);
 3574     __ mov(dst_reg, zr);
 3575   %}
 3576 
 3577   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3578     C2_MacroAssembler _masm(&cbuf);
 3579     Register dst_reg = as_Register($dst$$reg);
 3580     __ mov(dst_reg, (uint64_t)1);
 3581   %}
 3582 
 3583   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3584     C2_MacroAssembler _masm(&cbuf);
 3585     __ load_byte_map_base($dst$$Register);
 3586   %}
 3587 
 3588   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3589     C2_MacroAssembler _masm(&cbuf);
 3590     Register dst_reg = as_Register($dst$$reg);
 3591     address con = (address)$src$$constant;
 3592     if (con == NULL) {
 3593       ShouldNotReachHere();
 3594     } else {
 3595       relocInfo::relocType rtype = $src->constant_reloc();
 3596       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3597       __ set_narrow_oop(dst_reg, (jobject)con);
 3598     }
 3599   %}
 3600 
 3601   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3602     C2_MacroAssembler _masm(&cbuf);
 3603     Register dst_reg = as_Register($dst$$reg);
 3604     __ mov(dst_reg, zr);
 3605   %}
 3606 
 3607   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3608     C2_MacroAssembler _masm(&cbuf);
 3609     Register dst_reg = as_Register($dst$$reg);
 3610     address con = (address)$src$$constant;
 3611     if (con == NULL) {
 3612       ShouldNotReachHere();
 3613     } else {
 3614       relocInfo::relocType rtype = $src->constant_reloc();
 3615       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3616       __ set_narrow_klass(dst_reg, (Klass *)con);
 3617     }
 3618   %}
 3619 
 3620   // arithmetic encodings
 3621 
 3622   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3623     C2_MacroAssembler _masm(&cbuf);
 3624     Register dst_reg = as_Register($dst$$reg);
 3625     Register src_reg = as_Register($src1$$reg);
 3626     int32_t con = (int32_t)$src2$$constant;
 3627     // add has primary == 0, subtract has primary == 1
 3628     if ($primary) { con = -con; }
 3629     if (con < 0) {
 3630       __ subw(dst_reg, src_reg, -con);
 3631     } else {
 3632       __ addw(dst_reg, src_reg, con);
 3633     }
 3634   %}
 3635 
 3636   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3637     C2_MacroAssembler _masm(&cbuf);
 3638     Register dst_reg = as_Register($dst$$reg);
 3639     Register src_reg = as_Register($src1$$reg);
 3640     int32_t con = (int32_t)$src2$$constant;
 3641     // add has primary == 0, subtract has primary == 1
 3642     if ($primary) { con = -con; }
 3643     if (con < 0) {
 3644       __ sub(dst_reg, src_reg, -con);
 3645     } else {
 3646       __ add(dst_reg, src_reg, con);
 3647     }
 3648   %}
 3649 
 3650   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3651     C2_MacroAssembler _masm(&cbuf);
 3652    Register dst_reg = as_Register($dst$$reg);
 3653    Register src1_reg = as_Register($src1$$reg);
 3654    Register src2_reg = as_Register($src2$$reg);
 3655     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3656   %}
 3657 
 3658   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3659     C2_MacroAssembler _masm(&cbuf);
 3660    Register dst_reg = as_Register($dst$$reg);
 3661    Register src1_reg = as_Register($src1$$reg);
 3662    Register src2_reg = as_Register($src2$$reg);
 3663     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3664   %}
 3665 
 3666   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3667     C2_MacroAssembler _masm(&cbuf);
 3668    Register dst_reg = as_Register($dst$$reg);
 3669    Register src1_reg = as_Register($src1$$reg);
 3670    Register src2_reg = as_Register($src2$$reg);
 3671     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3672   %}
 3673 
 3674   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3675     C2_MacroAssembler _masm(&cbuf);
 3676    Register dst_reg = as_Register($dst$$reg);
 3677    Register src1_reg = as_Register($src1$$reg);
 3678    Register src2_reg = as_Register($src2$$reg);
 3679     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3680   %}
 3681 
 3682   // compare instruction encodings
 3683 
 3684   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3685     C2_MacroAssembler _masm(&cbuf);
 3686     Register reg1 = as_Register($src1$$reg);
 3687     Register reg2 = as_Register($src2$$reg);
 3688     __ cmpw(reg1, reg2);
 3689   %}
 3690 
 3691   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3692     C2_MacroAssembler _masm(&cbuf);
 3693     Register reg = as_Register($src1$$reg);
 3694     int32_t val = $src2$$constant;
 3695     if (val >= 0) {
 3696       __ subsw(zr, reg, val);
 3697     } else {
 3698       __ addsw(zr, reg, -val);
 3699     }
 3700   %}
 3701 
 3702   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3703     C2_MacroAssembler _masm(&cbuf);
 3704     Register reg1 = as_Register($src1$$reg);
 3705     uint32_t val = (uint32_t)$src2$$constant;
 3706     __ movw(rscratch1, val);
 3707     __ cmpw(reg1, rscratch1);
 3708   %}
 3709 
 3710   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3711     C2_MacroAssembler _masm(&cbuf);
 3712     Register reg1 = as_Register($src1$$reg);
 3713     Register reg2 = as_Register($src2$$reg);
 3714     __ cmp(reg1, reg2);
 3715   %}
 3716 
 3717   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3718     C2_MacroAssembler _masm(&cbuf);
 3719     Register reg = as_Register($src1$$reg);
 3720     int64_t val = $src2$$constant;
 3721     if (val >= 0) {
 3722       __ subs(zr, reg, val);
 3723     } else if (val != -val) {
 3724       __ adds(zr, reg, -val);
 3725     } else {
 3726     // aargh, Long.MIN_VALUE is a special case
 3727       __ orr(rscratch1, zr, (uint64_t)val);
 3728       __ subs(zr, reg, rscratch1);
 3729     }
 3730   %}
 3731 
 3732   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3733     C2_MacroAssembler _masm(&cbuf);
 3734     Register reg1 = as_Register($src1$$reg);
 3735     uint64_t val = (uint64_t)$src2$$constant;
 3736     __ mov(rscratch1, val);
 3737     __ cmp(reg1, rscratch1);
 3738   %}
 3739 
 3740   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3741     C2_MacroAssembler _masm(&cbuf);
 3742     Register reg1 = as_Register($src1$$reg);
 3743     Register reg2 = as_Register($src2$$reg);
 3744     __ cmp(reg1, reg2);
 3745   %}
 3746 
 3747   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3748     C2_MacroAssembler _masm(&cbuf);
 3749     Register reg1 = as_Register($src1$$reg);
 3750     Register reg2 = as_Register($src2$$reg);
 3751     __ cmpw(reg1, reg2);
 3752   %}
 3753 
 3754   enc_class aarch64_enc_testp(iRegP src) %{
 3755     C2_MacroAssembler _masm(&cbuf);
 3756     Register reg = as_Register($src$$reg);
 3757     __ cmp(reg, zr);
 3758   %}
 3759 
 3760   enc_class aarch64_enc_testn(iRegN src) %{
 3761     C2_MacroAssembler _masm(&cbuf);
 3762     Register reg = as_Register($src$$reg);
 3763     __ cmpw(reg, zr);
 3764   %}
 3765 
 3766   enc_class aarch64_enc_b(label lbl) %{
 3767     C2_MacroAssembler _masm(&cbuf);
 3768     Label *L = $lbl$$label;
 3769     __ b(*L);
 3770   %}
 3771 
 3772   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3773     C2_MacroAssembler _masm(&cbuf);
 3774     Label *L = $lbl$$label;
 3775     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3776   %}
 3777 
 3778   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3779     C2_MacroAssembler _masm(&cbuf);
 3780     Label *L = $lbl$$label;
 3781     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3782   %}
 3783 
 3784   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3785   %{
 3786      Register sub_reg = as_Register($sub$$reg);
 3787      Register super_reg = as_Register($super$$reg);
 3788      Register temp_reg = as_Register($temp$$reg);
 3789      Register result_reg = as_Register($result$$reg);
 3790 
 3791      Label miss;
 3792      C2_MacroAssembler _masm(&cbuf);
 3793      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3794                                      NULL, &miss,
 3795                                      /*set_cond_codes:*/ true);
 3796      if ($primary) {
 3797        __ mov(result_reg, zr);
 3798      }
 3799      __ bind(miss);
 3800   %}
 3801 
 3802   enc_class aarch64_enc_java_static_call(method meth) %{
 3803     C2_MacroAssembler _masm(&cbuf);
 3804 
 3805     address addr = (address)$meth$$method;
 3806     address call;
 3807     if (!_method) {
 3808       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3809       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3810       if (call == NULL) {
 3811         ciEnv::current()->record_failure("CodeCache is full");
 3812         return;
 3813       }
 3814     } else {
 3815       int method_index = resolved_method_index(cbuf);
 3816       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3817                                                   : static_call_Relocation::spec(method_index);
 3818       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3819       if (call == NULL) {
 3820         ciEnv::current()->record_failure("CodeCache is full");
 3821         return;
 3822       }
 3823       // Emit stub for static call
 3824       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3825       if (stub == NULL) {
 3826         ciEnv::current()->record_failure("CodeCache is full");
 3827         return;
 3828       }
 3829     }
 3830 
 3831     // Only non uncommon_trap calls need to reinitialize ptrue.
 3832     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3833       __ reinitialize_ptrue();
 3834     }
 3835   %}
 3836 
 3837   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3838     C2_MacroAssembler _masm(&cbuf);
 3839     int method_index = resolved_method_index(cbuf);
 3840     address call = __ ic_call((address)$meth$$method, method_index);
 3841     if (call == NULL) {
 3842       ciEnv::current()->record_failure("CodeCache is full");
 3843       return;
 3844     } else if (Compile::current()->max_vector_size() > 0) {
 3845       __ reinitialize_ptrue();
 3846     }
 3847   %}
 3848 
 3849   enc_class aarch64_enc_call_epilog() %{
 3850     C2_MacroAssembler _masm(&cbuf);
 3851     if (VerifyStackAtCalls) {
 3852       // Check that stack depth is unchanged: find majik cookie on stack
 3853       __ call_Unimplemented();
 3854     }
 3855   %}
 3856 
 3857   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3858     C2_MacroAssembler _masm(&cbuf);
 3859 
 3860     // some calls to generated routines (arraycopy code) are scheduled
 3861     // by C2 as runtime calls. if so we can call them using a br (they
 3862     // will be in a reachable segment) otherwise we have to use a blr
 3863     // which loads the absolute address into a register.
 3864     address entry = (address)$meth$$method;
 3865     CodeBlob *cb = CodeCache::find_blob(entry);
 3866     if (cb) {
 3867       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3868       if (call == NULL) {
 3869         ciEnv::current()->record_failure("CodeCache is full");
 3870         return;
 3871       }
 3872     } else {
 3873       Label retaddr;
 3874       __ adr(rscratch2, retaddr);
 3875       __ lea(rscratch1, RuntimeAddress(entry));
 3876       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3877       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3878       __ blr(rscratch1);
 3879       __ bind(retaddr);
 3880       __ add(sp, sp, 2 * wordSize);
 3881     }
 3882     if (Compile::current()->max_vector_size() > 0) {
 3883       __ reinitialize_ptrue();
 3884     }
 3885   %}
 3886 
 3887   enc_class aarch64_enc_rethrow() %{
 3888     C2_MacroAssembler _masm(&cbuf);
 3889     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3890   %}
 3891 
 3892   enc_class aarch64_enc_ret() %{
 3893     C2_MacroAssembler _masm(&cbuf);
 3894 #ifdef ASSERT
 3895     if (Compile::current()->max_vector_size() > 0) {
 3896       __ verify_ptrue();
 3897     }
 3898 #endif
 3899     __ ret(lr);
 3900   %}
 3901 
 3902   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3903     C2_MacroAssembler _masm(&cbuf);
 3904     Register target_reg = as_Register($jump_target$$reg);
 3905     __ br(target_reg);
 3906   %}
 3907 
 3908   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3909     C2_MacroAssembler _masm(&cbuf);
 3910     Register target_reg = as_Register($jump_target$$reg);
 3911     // exception oop should be in r0
 3912     // ret addr has been popped into lr
 3913     // callee expects it in r3
 3914     __ mov(r3, lr);
 3915     __ br(target_reg);
 3916   %}
 3917 
 3918   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3919     C2_MacroAssembler _masm(&cbuf);
 3920     Register oop = as_Register($object$$reg);
 3921     Register box = as_Register($box$$reg);
 3922     Register disp_hdr = as_Register($tmp$$reg);
 3923     Register tmp = as_Register($tmp2$$reg);
 3924     Label cont;
 3925     Label object_has_monitor;
 3926     Label cas_failed;
 3927 
 3928     assert_different_registers(oop, box, tmp, disp_hdr);
 3929 
 3930     // Load markWord from object into displaced_header.
 3931     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3932 
 3933     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3934       __ load_klass(tmp, oop);
 3935       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3936       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3937       __ br(Assembler::NE, cont);
 3938     }
 3939 
 3940     // Check for existing monitor
 3941     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3942 
 3943     if (!UseHeavyMonitors) {
 3944       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3945       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3946 
 3947       // Initialize the box. (Must happen before we update the object mark!)
 3948       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3949 
 3950       // Compare object markWord with an unlocked value (tmp) and if
 3951       // equal exchange the stack address of our box with object markWord.
 3952       // On failure disp_hdr contains the possibly locked markWord.
 3953       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3954                  /*release*/ true, /*weak*/ false, disp_hdr);
 3955       __ br(Assembler::EQ, cont);
 3956 
 3957       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3958 
 3959       // If the compare-and-exchange succeeded, then we found an unlocked
 3960       // object, will have now locked it will continue at label cont
 3961 
 3962       __ bind(cas_failed);
 3963       // We did not see an unlocked object so try the fast recursive case.
 3964 
 3965       // Check if the owner is self by comparing the value in the
 3966       // markWord of object (disp_hdr) with the stack pointer.
 3967       __ mov(rscratch1, sp);
 3968       __ sub(disp_hdr, disp_hdr, rscratch1);
 3969       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3970       // If condition is true we are cont and hence we can store 0 as the
 3971       // displaced header in the box, which indicates that it is a recursive lock.
 3972       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3973       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3974     } else {
 3975       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3976     }
 3977     __ b(cont);
 3978 
 3979     // Handle existing monitor.
 3980     __ bind(object_has_monitor);
 3981 
 3982     // The object's monitor m is unlocked iff m->owner == NULL,
 3983     // otherwise m->owner may contain a thread or a stack address.
 3984     //
 3985     // Try to CAS m->owner from NULL to current thread.
 3986     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3987     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3988                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3989 
 3990     // Store a non-null value into the box to avoid looking like a re-entrant
 3991     // lock. The fast-path monitor unlock code checks for
 3992     // markWord::monitor_value so use markWord::unused_mark which has the
 3993     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3994     __ mov(tmp, (address)markWord::unused_mark().value());
 3995     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3996 
 3997     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3998 
 3999     __ cmp(rscratch1, rthread);
 4000     __ br(Assembler::NE, cont); // Check for recursive locking
 4001 
 4002     // Recursive lock case
 4003     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 4004     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 4005 
 4006     __ bind(cont);
 4007     // flag == EQ indicates success
 4008     // flag == NE indicates failure
 4009   %}
 4010 
 4011   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 4012     C2_MacroAssembler _masm(&cbuf);
 4013     Register oop = as_Register($object$$reg);
 4014     Register box = as_Register($box$$reg);
 4015     Register disp_hdr = as_Register($tmp$$reg);
 4016     Register tmp = as_Register($tmp2$$reg);
 4017     Label cont;
 4018     Label object_has_monitor;
 4019 
 4020     assert_different_registers(oop, box, tmp, disp_hdr);
 4021 
 4022     if (!UseHeavyMonitors) {
 4023       // Find the lock address and load the displaced header from the stack.
 4024       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 4025 
 4026       // If the displaced header is 0, we have a recursive unlock.
 4027       __ cmp(disp_hdr, zr);
 4028       __ br(Assembler::EQ, cont);
 4029     }
 4030 
 4031     // Handle existing monitor.
 4032     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 4033     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 4034 
 4035     if (!UseHeavyMonitors) {
 4036       // Check if it is still a light weight lock, this is is true if we
 4037       // see the stack address of the basicLock in the markWord of the
 4038       // object.
 4039 
 4040       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 4041                  /*release*/ true, /*weak*/ false, tmp);
 4042     } else {
 4043       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 4044     }
 4045     __ b(cont);
 4046 
 4047     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 4048 
 4049     // Handle existing monitor.
 4050     __ bind(object_has_monitor);
 4051     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 4052     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 4053     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4054 
 4055     Label notRecursive;
 4056     __ cbz(disp_hdr, notRecursive);
 4057 
 4058     // Recursive lock
 4059     __ sub(disp_hdr, disp_hdr, 1u);
 4060     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4061     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 4062     __ b(cont);
 4063 
 4064     __ bind(notRecursive);
 4065     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 4066     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 4067     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 4068     __ cmp(rscratch1, zr); // Sets flags for result
 4069     __ cbnz(rscratch1, cont);
 4070     // need a release store here
 4071     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 4072     __ stlr(zr, tmp); // set unowned
 4073 
 4074     __ bind(cont);
 4075     // flag == EQ indicates success
 4076     // flag == NE indicates failure
 4077   %}
 4078 
 4079 %}
 4080 
 4081 //----------FRAME--------------------------------------------------------------
 4082 // Definition of frame structure and management information.
 4083 //
 4084 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4085 //                             |   (to get allocators register number
 4086 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4087 //  r   CALLER     |        |
 4088 //  o     |        +--------+      pad to even-align allocators stack-slot
 4089 //  w     V        |  pad0  |        numbers; owned by CALLER
 4090 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4091 //  h     ^        |   in   |  5
 4092 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4093 //  |     |        |        |  3
 4094 //  |     |        +--------+
 4095 //  V     |        | old out|      Empty on Intel, window on Sparc
 4096 //        |    old |preserve|      Must be even aligned.
 4097 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4098 //        |        |   in   |  3   area for Intel ret address
 4099 //     Owned by    |preserve|      Empty on Sparc.
 4100 //       SELF      +--------+
 4101 //        |        |  pad2  |  2   pad to align old SP
 4102 //        |        +--------+  1
 4103 //        |        | locks  |  0
 4104 //        |        +--------+----> OptoReg::stack0(), even aligned
 4105 //        |        |  pad1  | 11   pad to align new SP
 4106 //        |        +--------+
 4107 //        |        |        | 10
 4108 //        |        | spills |  9   spills
 4109 //        V        |        |  8   (pad0 slot for callee)
 4110 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4111 //        ^        |  out   |  7
 4112 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4113 //     Owned by    +--------+
 4114 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4115 //        |    new |preserve|      Must be even-aligned.
 4116 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4117 //        |        |        |
 4118 //
 4119 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4120 //         known from SELF's arguments and the Java calling convention.
 4121 //         Region 6-7 is determined per call site.
 4122 // Note 2: If the calling convention leaves holes in the incoming argument
 4123 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4124 //         are owned by the CALLEE.  Holes should not be necessary in the
 4125 //         incoming area, as the Java calling convention is completely under
 4126 //         the control of the AD file.  Doubles can be sorted and packed to
 4127 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 4128 //         varargs C calling conventions.
 4129 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4130 //         even aligned with pad0 as needed.
 4131 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4132 //           (the latter is true on Intel but is it false on AArch64?)
 4133 //         region 6-11 is even aligned; it may be padded out more so that
 4134 //         the region from SP to FP meets the minimum stack alignment.
 4135 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4136 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4137 //         SP meets the minimum alignment.
 4138 
 4139 frame %{
 4140   // These three registers define part of the calling convention
 4141   // between compiled code and the interpreter.
 4142 
 4143   // Inline Cache Register or Method for I2C.
 4144   inline_cache_reg(R12);
 4145 
 4146   // Number of stack slots consumed by locking an object
 4147   sync_stack_slots(2);
 4148 
 4149   // Compiled code's Frame Pointer
 4150   frame_pointer(R31);
 4151 
 4152   // Interpreter stores its frame pointer in a register which is
 4153   // stored to the stack by I2CAdaptors.
 4154   // I2CAdaptors convert from interpreted java to compiled java.
 4155   interpreter_frame_pointer(R29);
 4156 
 4157   // Stack alignment requirement
 4158   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4159 
 4160   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4161   // for calls to C.  Supports the var-args backing area for register parms.
 4162   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4163 
 4164   // The after-PROLOG location of the return address.  Location of
 4165   // return address specifies a type (REG or STACK) and a number
 4166   // representing the register number (i.e. - use a register name) or
 4167   // stack slot.
 4168   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4169   // Otherwise, it is above the locks and verification slot and alignment word
 4170   // TODO this may well be correct but need to check why that - 2 is there
 4171   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4172   // which folds in the space used for monitors
 4173   return_addr(STACK - 2 +
 4174               align_up((Compile::current()->in_preserve_stack_slots() +
 4175                         Compile::current()->fixed_slots()),
 4176                        stack_alignment_in_slots()));
 4177 
 4178   // Location of compiled Java return values.  Same as C for now.
 4179   return_value
 4180   %{
 4181     // TODO do we allow ideal_reg == Op_RegN???
 4182     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4183            "only return normal values");
 4184 
 4185     static const int lo[Op_RegL + 1] = { // enum name
 4186       0,                                 // Op_Node
 4187       0,                                 // Op_Set
 4188       R0_num,                            // Op_RegN
 4189       R0_num,                            // Op_RegI
 4190       R0_num,                            // Op_RegP
 4191       V0_num,                            // Op_RegF
 4192       V0_num,                            // Op_RegD
 4193       R0_num                             // Op_RegL
 4194     };
 4195 
 4196     static const int hi[Op_RegL + 1] = { // enum name
 4197       0,                                 // Op_Node
 4198       0,                                 // Op_Set
 4199       OptoReg::Bad,                      // Op_RegN
 4200       OptoReg::Bad,                      // Op_RegI
 4201       R0_H_num,                          // Op_RegP
 4202       OptoReg::Bad,                      // Op_RegF
 4203       V0_H_num,                          // Op_RegD
 4204       R0_H_num                           // Op_RegL
 4205     };
 4206 
 4207     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4208   %}
 4209 %}
 4210 
 4211 //----------ATTRIBUTES---------------------------------------------------------
 4212 //----------Operand Attributes-------------------------------------------------
 4213 op_attrib op_cost(1);        // Required cost attribute
 4214 
 4215 //----------Instruction Attributes---------------------------------------------
 4216 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4217 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4218 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4219                                 // a non-matching short branch variant
 4220                                 // of some long branch?
 4221 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4222                                 // be a power of 2) specifies the
 4223                                 // alignment that some part of the
 4224                                 // instruction (not necessarily the
 4225                                 // start) requires.  If > 1, a
 4226                                 // compute_padding() function must be
 4227                                 // provided for the instruction
 4228 
 4229 //----------OPERANDS-----------------------------------------------------------
 4230 // Operand definitions must precede instruction definitions for correct parsing
 4231 // in the ADLC because operands constitute user defined types which are used in
 4232 // instruction definitions.
 4233 
 4234 //----------Simple Operands----------------------------------------------------
 4235 
 4236 // Integer operands 32 bit
 4237 // 32 bit immediate
 4238 operand immI()
 4239 %{
 4240   match(ConI);
 4241 
 4242   op_cost(0);
 4243   format %{ %}
 4244   interface(CONST_INTER);
 4245 %}
 4246 
 4247 // 32 bit zero
 4248 operand immI0()
 4249 %{
 4250   predicate(n->get_int() == 0);
 4251   match(ConI);
 4252 
 4253   op_cost(0);
 4254   format %{ %}
 4255   interface(CONST_INTER);
 4256 %}
 4257 
 4258 // 32 bit unit increment
 4259 operand immI_1()
 4260 %{
 4261   predicate(n->get_int() == 1);
 4262   match(ConI);
 4263 
 4264   op_cost(0);
 4265   format %{ %}
 4266   interface(CONST_INTER);
 4267 %}
 4268 
 4269 // 32 bit unit decrement
 4270 operand immI_M1()
 4271 %{
 4272   predicate(n->get_int() == -1);
 4273   match(ConI);
 4274 
 4275   op_cost(0);
 4276   format %{ %}
 4277   interface(CONST_INTER);
 4278 %}
 4279 
 4280 // Shift values for add/sub extension shift
 4281 operand immIExt()
 4282 %{
 4283   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4284   match(ConI);
 4285 
 4286   op_cost(0);
 4287   format %{ %}
 4288   interface(CONST_INTER);
 4289 %}
 4290 
 4291 operand immI_gt_1()
 4292 %{
 4293   predicate(n->get_int() > 1);
 4294   match(ConI);
 4295 
 4296   op_cost(0);
 4297   format %{ %}
 4298   interface(CONST_INTER);
 4299 %}
 4300 
 4301 operand immI_le_4()
 4302 %{
 4303   predicate(n->get_int() <= 4);
 4304   match(ConI);
 4305 
 4306   op_cost(0);
 4307   format %{ %}
 4308   interface(CONST_INTER);
 4309 %}
 4310 
 4311 operand immI_31()
 4312 %{
 4313   predicate(n->get_int() == 31);
 4314   match(ConI);
 4315 
 4316   op_cost(0);
 4317   format %{ %}
 4318   interface(CONST_INTER);
 4319 %}
 4320 
 4321 operand immI_2()
 4322 %{
 4323   predicate(n->get_int() == 2);
 4324   match(ConI);
 4325 
 4326   op_cost(0);
 4327   format %{ %}
 4328   interface(CONST_INTER);
 4329 %}
 4330 
 4331 operand immI_4()
 4332 %{
 4333   predicate(n->get_int() == 4);
 4334   match(ConI);
 4335 
 4336   op_cost(0);
 4337   format %{ %}
 4338   interface(CONST_INTER);
 4339 %}
 4340 
 4341 operand immI_8()
 4342 %{
 4343   predicate(n->get_int() == 8);
 4344   match(ConI);
 4345 
 4346   op_cost(0);
 4347   format %{ %}
 4348   interface(CONST_INTER);
 4349 %}
 4350 
 4351 operand immI_16()
 4352 %{
 4353   predicate(n->get_int() == 16);
 4354   match(ConI);
 4355 
 4356   op_cost(0);
 4357   format %{ %}
 4358   interface(CONST_INTER);
 4359 %}
 4360 
 4361 operand immI_24()
 4362 %{
 4363   predicate(n->get_int() == 24);
 4364   match(ConI);
 4365 
 4366   op_cost(0);
 4367   format %{ %}
 4368   interface(CONST_INTER);
 4369 %}
 4370 
 4371 operand immI_32()
 4372 %{
 4373   predicate(n->get_int() == 32);
 4374   match(ConI);
 4375 
 4376   op_cost(0);
 4377   format %{ %}
 4378   interface(CONST_INTER);
 4379 %}
 4380 
 4381 operand immI_48()
 4382 %{
 4383   predicate(n->get_int() == 48);
 4384   match(ConI);
 4385 
 4386   op_cost(0);
 4387   format %{ %}
 4388   interface(CONST_INTER);
 4389 %}
 4390 
 4391 operand immI_56()
 4392 %{
 4393   predicate(n->get_int() == 56);
 4394   match(ConI);
 4395 
 4396   op_cost(0);
 4397   format %{ %}
 4398   interface(CONST_INTER);
 4399 %}
 4400 
 4401 operand immI_63()
 4402 %{
 4403   predicate(n->get_int() == 63);
 4404   match(ConI);
 4405 
 4406   op_cost(0);
 4407   format %{ %}
 4408   interface(CONST_INTER);
 4409 %}
 4410 
 4411 operand immI_64()
 4412 %{
 4413   predicate(n->get_int() == 64);
 4414   match(ConI);
 4415 
 4416   op_cost(0);
 4417   format %{ %}
 4418   interface(CONST_INTER);
 4419 %}
 4420 
 4421 operand immI_255()
 4422 %{
 4423   predicate(n->get_int() == 255);
 4424   match(ConI);
 4425 
 4426   op_cost(0);
 4427   format %{ %}
 4428   interface(CONST_INTER);
 4429 %}
 4430 
 4431 operand immI_65535()
 4432 %{
 4433   predicate(n->get_int() == 65535);
 4434   match(ConI);
 4435 
 4436   op_cost(0);
 4437   format %{ %}
 4438   interface(CONST_INTER);
 4439 %}
 4440 
 4441 operand immL_255()
 4442 %{
 4443   predicate(n->get_long() == 255L);
 4444   match(ConL);
 4445 
 4446   op_cost(0);
 4447   format %{ %}
 4448   interface(CONST_INTER);
 4449 %}
 4450 
 4451 operand immL_65535()
 4452 %{
 4453   predicate(n->get_long() == 65535L);
 4454   match(ConL);
 4455 
 4456   op_cost(0);
 4457   format %{ %}
 4458   interface(CONST_INTER);
 4459 %}
 4460 
 4461 operand immL_4294967295()
 4462 %{
 4463   predicate(n->get_long() == 4294967295L);
 4464   match(ConL);
 4465 
 4466   op_cost(0);
 4467   format %{ %}
 4468   interface(CONST_INTER);
 4469 %}
 4470 
 4471 operand immL_bitmask()
 4472 %{
 4473   predicate((n->get_long() != 0)
 4474             && ((n->get_long() & 0xc000000000000000l) == 0)
 4475             && is_power_of_2(n->get_long() + 1));
 4476   match(ConL);
 4477 
 4478   op_cost(0);
 4479   format %{ %}
 4480   interface(CONST_INTER);
 4481 %}
 4482 
 4483 operand immI_bitmask()
 4484 %{
 4485   predicate((n->get_int() != 0)
 4486             && ((n->get_int() & 0xc0000000) == 0)
 4487             && is_power_of_2(n->get_int() + 1));
 4488   match(ConI);
 4489 
 4490   op_cost(0);
 4491   format %{ %}
 4492   interface(CONST_INTER);
 4493 %}
 4494 
 4495 operand immL_positive_bitmaskI()
 4496 %{
 4497   predicate((n->get_long() != 0)
 4498             && ((julong)n->get_long() < 0x80000000ULL)
 4499             && is_power_of_2(n->get_long() + 1));
 4500   match(ConL);
 4501 
 4502   op_cost(0);
 4503   format %{ %}
 4504   interface(CONST_INTER);
 4505 %}
 4506 
 4507 // Scale values for scaled offset addressing modes (up to long but not quad)
 4508 operand immIScale()
 4509 %{
 4510   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4511   match(ConI);
 4512 
 4513   op_cost(0);
 4514   format %{ %}
 4515   interface(CONST_INTER);
 4516 %}
 4517 
 4518 // 26 bit signed offset -- for pc-relative branches
 4519 operand immI26()
 4520 %{
 4521   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4522   match(ConI);
 4523 
 4524   op_cost(0);
 4525   format %{ %}
 4526   interface(CONST_INTER);
 4527 %}
 4528 
 4529 // 19 bit signed offset -- for pc-relative loads
 4530 operand immI19()
 4531 %{
 4532   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4533   match(ConI);
 4534 
 4535   op_cost(0);
 4536   format %{ %}
 4537   interface(CONST_INTER);
 4538 %}
 4539 
 4540 // 12 bit unsigned offset -- for base plus immediate loads
 4541 operand immIU12()
 4542 %{
 4543   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4544   match(ConI);
 4545 
 4546   op_cost(0);
 4547   format %{ %}
 4548   interface(CONST_INTER);
 4549 %}
 4550 
 4551 operand immLU12()
 4552 %{
 4553   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4554   match(ConL);
 4555 
 4556   op_cost(0);
 4557   format %{ %}
 4558   interface(CONST_INTER);
 4559 %}
 4560 
 4561 // Offset for scaled or unscaled immediate loads and stores
 4562 operand immIOffset()
 4563 %{
 4564   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4565   match(ConI);
 4566 
 4567   op_cost(0);
 4568   format %{ %}
 4569   interface(CONST_INTER);
 4570 %}
 4571 
 4572 operand immIOffset1()
 4573 %{
 4574   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4575   match(ConI);
 4576 
 4577   op_cost(0);
 4578   format %{ %}
 4579   interface(CONST_INTER);
 4580 %}
 4581 
 4582 operand immIOffset2()
 4583 %{
 4584   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4585   match(ConI);
 4586 
 4587   op_cost(0);
 4588   format %{ %}
 4589   interface(CONST_INTER);
 4590 %}
 4591 
 4592 operand immIOffset4()
 4593 %{
 4594   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4595   match(ConI);
 4596 
 4597   op_cost(0);
 4598   format %{ %}
 4599   interface(CONST_INTER);
 4600 %}
 4601 
 4602 operand immIOffset8()
 4603 %{
 4604   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4605   match(ConI);
 4606 
 4607   op_cost(0);
 4608   format %{ %}
 4609   interface(CONST_INTER);
 4610 %}
 4611 
 4612 operand immIOffset16()
 4613 %{
 4614   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4615   match(ConI);
 4616 
 4617   op_cost(0);
 4618   format %{ %}
 4619   interface(CONST_INTER);
 4620 %}
 4621 
 4622 operand immLoffset()
 4623 %{
 4624   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4625   match(ConL);
 4626 
 4627   op_cost(0);
 4628   format %{ %}
 4629   interface(CONST_INTER);
 4630 %}
 4631 
 4632 operand immLoffset1()
 4633 %{
 4634   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4635   match(ConL);
 4636 
 4637   op_cost(0);
 4638   format %{ %}
 4639   interface(CONST_INTER);
 4640 %}
 4641 
 4642 operand immLoffset2()
 4643 %{
 4644   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4645   match(ConL);
 4646 
 4647   op_cost(0);
 4648   format %{ %}
 4649   interface(CONST_INTER);
 4650 %}
 4651 
 4652 operand immLoffset4()
 4653 %{
 4654   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4655   match(ConL);
 4656 
 4657   op_cost(0);
 4658   format %{ %}
 4659   interface(CONST_INTER);
 4660 %}
 4661 
 4662 operand immLoffset8()
 4663 %{
 4664   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4665   match(ConL);
 4666 
 4667   op_cost(0);
 4668   format %{ %}
 4669   interface(CONST_INTER);
 4670 %}
 4671 
 4672 operand immLoffset16()
 4673 %{
 4674   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4675   match(ConL);
 4676 
 4677   op_cost(0);
 4678   format %{ %}
 4679   interface(CONST_INTER);
 4680 %}
 4681 
 4682 // 8 bit signed value.
 4683 operand immI8()
 4684 %{
 4685   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4686   match(ConI);
 4687 
 4688   op_cost(0);
 4689   format %{ %}
 4690   interface(CONST_INTER);
 4691 %}
 4692 
 4693 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4694 operand immI8_shift8()
 4695 %{
 4696   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4697             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4698   match(ConI);
 4699 
 4700   op_cost(0);
 4701   format %{ %}
 4702   interface(CONST_INTER);
 4703 %}
 4704 
 4705 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4706 operand immL8_shift8()
 4707 %{
 4708   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4709             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4710   match(ConL);
 4711 
 4712   op_cost(0);
 4713   format %{ %}
 4714   interface(CONST_INTER);
 4715 %}
 4716 
 4717 // 8 bit integer valid for vector add sub immediate
 4718 operand immBAddSubV()
 4719 %{
 4720   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4721   match(ConI);
 4722 
 4723   op_cost(0);
 4724   format %{ %}
 4725   interface(CONST_INTER);
 4726 %}
 4727 
 4728 // 32 bit integer valid for add sub immediate
 4729 operand immIAddSub()
 4730 %{
 4731   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4732   match(ConI);
 4733   op_cost(0);
 4734   format %{ %}
 4735   interface(CONST_INTER);
 4736 %}
 4737 
 4738 // 32 bit integer valid for vector add sub immediate
 4739 operand immIAddSubV()
 4740 %{
 4741   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4742   match(ConI);
 4743 
 4744   op_cost(0);
 4745   format %{ %}
 4746   interface(CONST_INTER);
 4747 %}
 4748 
 4749 // 32 bit unsigned integer valid for logical immediate
 4750 
 4751 operand immBLog()
 4752 %{
 4753   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4754   match(ConI);
 4755 
 4756   op_cost(0);
 4757   format %{ %}
 4758   interface(CONST_INTER);
 4759 %}
 4760 
 4761 operand immSLog()
 4762 %{
 4763   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4764   match(ConI);
 4765 
 4766   op_cost(0);
 4767   format %{ %}
 4768   interface(CONST_INTER);
 4769 %}
 4770 
 4771 operand immILog()
 4772 %{
 4773   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4774   match(ConI);
 4775 
 4776   op_cost(0);
 4777   format %{ %}
 4778   interface(CONST_INTER);
 4779 %}
 4780 
 4781 // Integer operands 64 bit
 4782 // 64 bit immediate
 4783 operand immL()
 4784 %{
 4785   match(ConL);
 4786 
 4787   op_cost(0);
 4788   format %{ %}
 4789   interface(CONST_INTER);
 4790 %}
 4791 
 4792 // 64 bit zero
 4793 operand immL0()
 4794 %{
 4795   predicate(n->get_long() == 0);
 4796   match(ConL);
 4797 
 4798   op_cost(0);
 4799   format %{ %}
 4800   interface(CONST_INTER);
 4801 %}
 4802 
 4803 // 64 bit unit increment
 4804 operand immL_1()
 4805 %{
 4806   predicate(n->get_long() == 1);
 4807   match(ConL);
 4808 
 4809   op_cost(0);
 4810   format %{ %}
 4811   interface(CONST_INTER);
 4812 %}
 4813 
 4814 // 64 bit unit decrement
 4815 operand immL_M1()
 4816 %{
 4817   predicate(n->get_long() == -1);
 4818   match(ConL);
 4819 
 4820   op_cost(0);
 4821   format %{ %}
 4822   interface(CONST_INTER);
 4823 %}
 4824 
 4825 // 32 bit offset of pc in thread anchor
 4826 
 4827 operand immL_pc_off()
 4828 %{
 4829   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4830                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4831   match(ConL);
 4832 
 4833   op_cost(0);
 4834   format %{ %}
 4835   interface(CONST_INTER);
 4836 %}
 4837 
 4838 // 64 bit integer valid for add sub immediate
 4839 operand immLAddSub()
 4840 %{
 4841   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4842   match(ConL);
 4843   op_cost(0);
 4844   format %{ %}
 4845   interface(CONST_INTER);
 4846 %}
 4847 
 4848 // 64 bit integer valid for addv subv immediate
 4849 operand immLAddSubV()
 4850 %{
 4851   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4852   match(ConL);
 4853 
 4854   op_cost(0);
 4855   format %{ %}
 4856   interface(CONST_INTER);
 4857 %}
 4858 
 4859 // 64 bit integer valid for logical immediate
 4860 operand immLLog()
 4861 %{
 4862   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4863   match(ConL);
 4864   op_cost(0);
 4865   format %{ %}
 4866   interface(CONST_INTER);
 4867 %}
 4868 
 4869 // Long Immediate: low 32-bit mask
 4870 operand immL_32bits()
 4871 %{
 4872   predicate(n->get_long() == 0xFFFFFFFFL);
 4873   match(ConL);
 4874   op_cost(0);
 4875   format %{ %}
 4876   interface(CONST_INTER);
 4877 %}
 4878 
 4879 // Pointer operands
 4880 // Pointer Immediate
 4881 operand immP()
 4882 %{
 4883   match(ConP);
 4884 
 4885   op_cost(0);
 4886   format %{ %}
 4887   interface(CONST_INTER);
 4888 %}
 4889 
 4890 // NULL Pointer Immediate
 4891 operand immP0()
 4892 %{
 4893   predicate(n->get_ptr() == 0);
 4894   match(ConP);
 4895 
 4896   op_cost(0);
 4897   format %{ %}
 4898   interface(CONST_INTER);
 4899 %}
 4900 
 4901 // Pointer Immediate One
 4902 // this is used in object initialization (initial object header)
 4903 operand immP_1()
 4904 %{
 4905   predicate(n->get_ptr() == 1);
 4906   match(ConP);
 4907 
 4908   op_cost(0);
 4909   format %{ %}
 4910   interface(CONST_INTER);
 4911 %}
 4912 
 4913 // Card Table Byte Map Base
 4914 operand immByteMapBase()
 4915 %{
 4916   // Get base of card map
 4917   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4918             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4919   match(ConP);
 4920 
 4921   op_cost(0);
 4922   format %{ %}
 4923   interface(CONST_INTER);
 4924 %}
 4925 
 4926 // Pointer Immediate Minus One
 4927 // this is used when we want to write the current PC to the thread anchor
 4928 operand immP_M1()
 4929 %{
 4930   predicate(n->get_ptr() == -1);
 4931   match(ConP);
 4932 
 4933   op_cost(0);
 4934   format %{ %}
 4935   interface(CONST_INTER);
 4936 %}
 4937 
 4938 // Pointer Immediate Minus Two
 4939 // this is used when we want to write the current PC to the thread anchor
 4940 operand immP_M2()
 4941 %{
 4942   predicate(n->get_ptr() == -2);
 4943   match(ConP);
 4944 
 4945   op_cost(0);
 4946   format %{ %}
 4947   interface(CONST_INTER);
 4948 %}
 4949 
 4950 // Float and Double operands
 4951 // Double Immediate
 4952 operand immD()
 4953 %{
 4954   match(ConD);
 4955   op_cost(0);
 4956   format %{ %}
 4957   interface(CONST_INTER);
 4958 %}
 4959 
 4960 // Double Immediate: +0.0d
 4961 operand immD0()
 4962 %{
 4963   predicate(jlong_cast(n->getd()) == 0);
 4964   match(ConD);
 4965 
 4966   op_cost(0);
 4967   format %{ %}
 4968   interface(CONST_INTER);
 4969 %}
 4970 
 4971 // constant 'double +0.0'.
 4972 operand immDPacked()
 4973 %{
 4974   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4975   match(ConD);
 4976   op_cost(0);
 4977   format %{ %}
 4978   interface(CONST_INTER);
 4979 %}
 4980 
 4981 // Float Immediate
 4982 operand immF()
 4983 %{
 4984   match(ConF);
 4985   op_cost(0);
 4986   format %{ %}
 4987   interface(CONST_INTER);
 4988 %}
 4989 
 4990 // Float Immediate: +0.0f.
 4991 operand immF0()
 4992 %{
 4993   predicate(jint_cast(n->getf()) == 0);
 4994   match(ConF);
 4995 
 4996   op_cost(0);
 4997   format %{ %}
 4998   interface(CONST_INTER);
 4999 %}
 5000 
 5001 //
 5002 operand immFPacked()
 5003 %{
 5004   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 5005   match(ConF);
 5006   op_cost(0);
 5007   format %{ %}
 5008   interface(CONST_INTER);
 5009 %}
 5010 
 5011 // Narrow pointer operands
 5012 // Narrow Pointer Immediate
 5013 operand immN()
 5014 %{
 5015   match(ConN);
 5016 
 5017   op_cost(0);
 5018   format %{ %}
 5019   interface(CONST_INTER);
 5020 %}
 5021 
 5022 // Narrow NULL Pointer Immediate
 5023 operand immN0()
 5024 %{
 5025   predicate(n->get_narrowcon() == 0);
 5026   match(ConN);
 5027 
 5028   op_cost(0);
 5029   format %{ %}
 5030   interface(CONST_INTER);
 5031 %}
 5032 
 5033 operand immNKlass()
 5034 %{
 5035   match(ConNKlass);
 5036 
 5037   op_cost(0);
 5038   format %{ %}
 5039   interface(CONST_INTER);
 5040 %}
 5041 
 5042 // Integer 32 bit Register Operands
 5043 // Integer 32 bitRegister (excludes SP)
 5044 operand iRegI()
 5045 %{
 5046   constraint(ALLOC_IN_RC(any_reg32));
 5047   match(RegI);
 5048   match(iRegINoSp);
 5049   op_cost(0);
 5050   format %{ %}
 5051   interface(REG_INTER);
 5052 %}
 5053 
 5054 // Integer 32 bit Register not Special
 5055 operand iRegINoSp()
 5056 %{
 5057   constraint(ALLOC_IN_RC(no_special_reg32));
 5058   match(RegI);
 5059   op_cost(0);
 5060   format %{ %}
 5061   interface(REG_INTER);
 5062 %}
 5063 
 5064 // Integer 64 bit Register Operands
 5065 // Integer 64 bit Register (includes SP)
 5066 operand iRegL()
 5067 %{
 5068   constraint(ALLOC_IN_RC(any_reg));
 5069   match(RegL);
 5070   match(iRegLNoSp);
 5071   op_cost(0);
 5072   format %{ %}
 5073   interface(REG_INTER);
 5074 %}
 5075 
 5076 // Integer 64 bit Register not Special
 5077 operand iRegLNoSp()
 5078 %{
 5079   constraint(ALLOC_IN_RC(no_special_reg));
 5080   match(RegL);
 5081   match(iRegL_R0);
 5082   format %{ %}
 5083   interface(REG_INTER);
 5084 %}
 5085 
 5086 // Pointer Register Operands
 5087 // Pointer Register
 5088 operand iRegP()
 5089 %{
 5090   constraint(ALLOC_IN_RC(ptr_reg));
 5091   match(RegP);
 5092   match(iRegPNoSp);
 5093   match(iRegP_R0);
 5094   //match(iRegP_R2);
 5095   //match(iRegP_R4);
 5096   //match(iRegP_R5);
 5097   match(thread_RegP);
 5098   op_cost(0);
 5099   format %{ %}
 5100   interface(REG_INTER);
 5101 %}
 5102 
 5103 // Pointer 64 bit Register not Special
 5104 operand iRegPNoSp()
 5105 %{
 5106   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 5107   match(RegP);
 5108   // match(iRegP);
 5109   // match(iRegP_R0);
 5110   // match(iRegP_R2);
 5111   // match(iRegP_R4);
 5112   // match(iRegP_R5);
 5113   // match(thread_RegP);
 5114   op_cost(0);
 5115   format %{ %}
 5116   interface(REG_INTER);
 5117 %}
 5118 
 5119 // Pointer 64 bit Register R0 only
 5120 operand iRegP_R0()
 5121 %{
 5122   constraint(ALLOC_IN_RC(r0_reg));
 5123   match(RegP);
 5124   // match(iRegP);
 5125   match(iRegPNoSp);
 5126   op_cost(0);
 5127   format %{ %}
 5128   interface(REG_INTER);
 5129 %}
 5130 
 5131 // Pointer 64 bit Register R1 only
 5132 operand iRegP_R1()
 5133 %{
 5134   constraint(ALLOC_IN_RC(r1_reg));
 5135   match(RegP);
 5136   // match(iRegP);
 5137   match(iRegPNoSp);
 5138   op_cost(0);
 5139   format %{ %}
 5140   interface(REG_INTER);
 5141 %}
 5142 
 5143 // Pointer 64 bit Register R2 only
 5144 operand iRegP_R2()
 5145 %{
 5146   constraint(ALLOC_IN_RC(r2_reg));
 5147   match(RegP);
 5148   // match(iRegP);
 5149   match(iRegPNoSp);
 5150   op_cost(0);
 5151   format %{ %}
 5152   interface(REG_INTER);
 5153 %}
 5154 
 5155 // Pointer 64 bit Register R3 only
 5156 operand iRegP_R3()
 5157 %{
 5158   constraint(ALLOC_IN_RC(r3_reg));
 5159   match(RegP);
 5160   // match(iRegP);
 5161   match(iRegPNoSp);
 5162   op_cost(0);
 5163   format %{ %}
 5164   interface(REG_INTER);
 5165 %}
 5166 
 5167 // Pointer 64 bit Register R4 only
 5168 operand iRegP_R4()
 5169 %{
 5170   constraint(ALLOC_IN_RC(r4_reg));
 5171   match(RegP);
 5172   // match(iRegP);
 5173   match(iRegPNoSp);
 5174   op_cost(0);
 5175   format %{ %}
 5176   interface(REG_INTER);
 5177 %}
 5178 
 5179 // Pointer 64 bit Register R5 only
 5180 operand iRegP_R5()
 5181 %{
 5182   constraint(ALLOC_IN_RC(r5_reg));
 5183   match(RegP);
 5184   // match(iRegP);
 5185   match(iRegPNoSp);
 5186   op_cost(0);
 5187   format %{ %}
 5188   interface(REG_INTER);
 5189 %}
 5190 
 5191 // Pointer 64 bit Register R10 only
 5192 operand iRegP_R10()
 5193 %{
 5194   constraint(ALLOC_IN_RC(r10_reg));
 5195   match(RegP);
 5196   // match(iRegP);
 5197   match(iRegPNoSp);
 5198   op_cost(0);
 5199   format %{ %}
 5200   interface(REG_INTER);
 5201 %}
 5202 
 5203 // Long 64 bit Register R0 only
 5204 operand iRegL_R0()
 5205 %{
 5206   constraint(ALLOC_IN_RC(r0_reg));
 5207   match(RegL);
 5208   match(iRegLNoSp);
 5209   op_cost(0);
 5210   format %{ %}
 5211   interface(REG_INTER);
 5212 %}
 5213 
 5214 // Long 64 bit Register R2 only
 5215 operand iRegL_R2()
 5216 %{
 5217   constraint(ALLOC_IN_RC(r2_reg));
 5218   match(RegL);
 5219   match(iRegLNoSp);
 5220   op_cost(0);
 5221   format %{ %}
 5222   interface(REG_INTER);
 5223 %}
 5224 
 5225 // Long 64 bit Register R3 only
 5226 operand iRegL_R3()
 5227 %{
 5228   constraint(ALLOC_IN_RC(r3_reg));
 5229   match(RegL);
 5230   match(iRegLNoSp);
 5231   op_cost(0);
 5232   format %{ %}
 5233   interface(REG_INTER);
 5234 %}
 5235 
 5236 // Long 64 bit Register R11 only
 5237 operand iRegL_R11()
 5238 %{
 5239   constraint(ALLOC_IN_RC(r11_reg));
 5240   match(RegL);
 5241   match(iRegLNoSp);
 5242   op_cost(0);
 5243   format %{ %}
 5244   interface(REG_INTER);
 5245 %}
 5246 
 5247 // Pointer 64 bit Register FP only
 5248 operand iRegP_FP()
 5249 %{
 5250   constraint(ALLOC_IN_RC(fp_reg));
 5251   match(RegP);
 5252   // match(iRegP);
 5253   op_cost(0);
 5254   format %{ %}
 5255   interface(REG_INTER);
 5256 %}
 5257 
 5258 // Register R0 only
 5259 operand iRegI_R0()
 5260 %{
 5261   constraint(ALLOC_IN_RC(int_r0_reg));
 5262   match(RegI);
 5263   match(iRegINoSp);
 5264   op_cost(0);
 5265   format %{ %}
 5266   interface(REG_INTER);
 5267 %}
 5268 
 5269 // Register R2 only
 5270 operand iRegI_R2()
 5271 %{
 5272   constraint(ALLOC_IN_RC(int_r2_reg));
 5273   match(RegI);
 5274   match(iRegINoSp);
 5275   op_cost(0);
 5276   format %{ %}
 5277   interface(REG_INTER);
 5278 %}
 5279 
 5280 // Register R3 only
 5281 operand iRegI_R3()
 5282 %{
 5283   constraint(ALLOC_IN_RC(int_r3_reg));
 5284   match(RegI);
 5285   match(iRegINoSp);
 5286   op_cost(0);
 5287   format %{ %}
 5288   interface(REG_INTER);
 5289 %}
 5290 
 5291 
 5292 // Register R4 only
 5293 operand iRegI_R4()
 5294 %{
 5295   constraint(ALLOC_IN_RC(int_r4_reg));
 5296   match(RegI);
 5297   match(iRegINoSp);
 5298   op_cost(0);
 5299   format %{ %}
 5300   interface(REG_INTER);
 5301 %}
 5302 
 5303 
 5304 // Pointer Register Operands
 5305 // Narrow Pointer Register
 5306 operand iRegN()
 5307 %{
 5308   constraint(ALLOC_IN_RC(any_reg32));
 5309   match(RegN);
 5310   match(iRegNNoSp);
 5311   op_cost(0);
 5312   format %{ %}
 5313   interface(REG_INTER);
 5314 %}
 5315 
 5316 operand iRegN_R0()
 5317 %{
 5318   constraint(ALLOC_IN_RC(r0_reg));
 5319   match(iRegN);
 5320   op_cost(0);
 5321   format %{ %}
 5322   interface(REG_INTER);
 5323 %}
 5324 
 5325 operand iRegN_R2()
 5326 %{
 5327   constraint(ALLOC_IN_RC(r2_reg));
 5328   match(iRegN);
 5329   op_cost(0);
 5330   format %{ %}
 5331   interface(REG_INTER);
 5332 %}
 5333 
 5334 operand iRegN_R3()
 5335 %{
 5336   constraint(ALLOC_IN_RC(r3_reg));
 5337   match(iRegN);
 5338   op_cost(0);
 5339   format %{ %}
 5340   interface(REG_INTER);
 5341 %}
 5342 
 5343 // Integer 64 bit Register not Special
 5344 operand iRegNNoSp()
 5345 %{
 5346   constraint(ALLOC_IN_RC(no_special_reg32));
 5347   match(RegN);
 5348   op_cost(0);
 5349   format %{ %}
 5350   interface(REG_INTER);
 5351 %}
 5352 
 5353 // heap base register -- used for encoding immN0
 5354 
 5355 operand iRegIHeapbase()
 5356 %{
 5357   constraint(ALLOC_IN_RC(heapbase_reg));
 5358   match(RegI);
 5359   op_cost(0);
 5360   format %{ %}
 5361   interface(REG_INTER);
 5362 %}
 5363 
 5364 // Float Register
 5365 // Float register operands
 5366 operand vRegF()
 5367 %{
 5368   constraint(ALLOC_IN_RC(float_reg));
 5369   match(RegF);
 5370 
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 // Double Register
 5377 // Double register operands
 5378 operand vRegD()
 5379 %{
 5380   constraint(ALLOC_IN_RC(double_reg));
 5381   match(RegD);
 5382 
 5383   op_cost(0);
 5384   format %{ %}
 5385   interface(REG_INTER);
 5386 %}
 5387 
 5388 // Generic vector class. This will be used for
 5389 // all vector operands, including NEON and SVE,
 5390 // but currently only used for SVE VecA.
 5391 operand vReg()
 5392 %{
 5393   constraint(ALLOC_IN_RC(vectora_reg));
 5394   match(VecA);
 5395   op_cost(0);
 5396   format %{ %}
 5397   interface(REG_INTER);
 5398 %}
 5399 
 5400 operand vecD()
 5401 %{
 5402   constraint(ALLOC_IN_RC(vectord_reg));
 5403   match(VecD);
 5404 
 5405   op_cost(0);
 5406   format %{ %}
 5407   interface(REG_INTER);
 5408 %}
 5409 
 5410 operand vecX()
 5411 %{
 5412   constraint(ALLOC_IN_RC(vectorx_reg));
 5413   match(VecX);
 5414 
 5415   op_cost(0);
 5416   format %{ %}
 5417   interface(REG_INTER);
 5418 %}
 5419 
 5420 operand vRegD_V0()
 5421 %{
 5422   constraint(ALLOC_IN_RC(v0_reg));
 5423   match(RegD);
 5424   op_cost(0);
 5425   format %{ %}
 5426   interface(REG_INTER);
 5427 %}
 5428 
 5429 operand vRegD_V1()
 5430 %{
 5431   constraint(ALLOC_IN_RC(v1_reg));
 5432   match(RegD);
 5433   op_cost(0);
 5434   format %{ %}
 5435   interface(REG_INTER);
 5436 %}
 5437 
 5438 operand vRegD_V2()
 5439 %{
 5440   constraint(ALLOC_IN_RC(v2_reg));
 5441   match(RegD);
 5442   op_cost(0);
 5443   format %{ %}
 5444   interface(REG_INTER);
 5445 %}
 5446 
 5447 operand vRegD_V3()
 5448 %{
 5449   constraint(ALLOC_IN_RC(v3_reg));
 5450   match(RegD);
 5451   op_cost(0);
 5452   format %{ %}
 5453   interface(REG_INTER);
 5454 %}
 5455 
 5456 operand vRegD_V4()
 5457 %{
 5458   constraint(ALLOC_IN_RC(v4_reg));
 5459   match(RegD);
 5460   op_cost(0);
 5461   format %{ %}
 5462   interface(REG_INTER);
 5463 %}
 5464 
 5465 operand vRegD_V5()
 5466 %{
 5467   constraint(ALLOC_IN_RC(v5_reg));
 5468   match(RegD);
 5469   op_cost(0);
 5470   format %{ %}
 5471   interface(REG_INTER);
 5472 %}
 5473 
 5474 operand vRegD_V6()
 5475 %{
 5476   constraint(ALLOC_IN_RC(v6_reg));
 5477   match(RegD);
 5478   op_cost(0);
 5479   format %{ %}
 5480   interface(REG_INTER);
 5481 %}
 5482 
 5483 operand vRegD_V7()
 5484 %{
 5485   constraint(ALLOC_IN_RC(v7_reg));
 5486   match(RegD);
 5487   op_cost(0);
 5488   format %{ %}
 5489   interface(REG_INTER);
 5490 %}
 5491 
 5492 operand vRegD_V8()
 5493 %{
 5494   constraint(ALLOC_IN_RC(v8_reg));
 5495   match(RegD);
 5496   op_cost(0);
 5497   format %{ %}
 5498   interface(REG_INTER);
 5499 %}
 5500 
 5501 operand vRegD_V9()
 5502 %{
 5503   constraint(ALLOC_IN_RC(v9_reg));
 5504   match(RegD);
 5505   op_cost(0);
 5506   format %{ %}
 5507   interface(REG_INTER);
 5508 %}
 5509 
 5510 operand vRegD_V10()
 5511 %{
 5512   constraint(ALLOC_IN_RC(v10_reg));
 5513   match(RegD);
 5514   op_cost(0);
 5515   format %{ %}
 5516   interface(REG_INTER);
 5517 %}
 5518 
 5519 operand vRegD_V11()
 5520 %{
 5521   constraint(ALLOC_IN_RC(v11_reg));
 5522   match(RegD);
 5523   op_cost(0);
 5524   format %{ %}
 5525   interface(REG_INTER);
 5526 %}
 5527 
 5528 operand vRegD_V12()
 5529 %{
 5530   constraint(ALLOC_IN_RC(v12_reg));
 5531   match(RegD);
 5532   op_cost(0);
 5533   format %{ %}
 5534   interface(REG_INTER);
 5535 %}
 5536 
 5537 operand vRegD_V13()
 5538 %{
 5539   constraint(ALLOC_IN_RC(v13_reg));
 5540   match(RegD);
 5541   op_cost(0);
 5542   format %{ %}
 5543   interface(REG_INTER);
 5544 %}
 5545 
 5546 operand vRegD_V14()
 5547 %{
 5548   constraint(ALLOC_IN_RC(v14_reg));
 5549   match(RegD);
 5550   op_cost(0);
 5551   format %{ %}
 5552   interface(REG_INTER);
 5553 %}
 5554 
 5555 operand vRegD_V15()
 5556 %{
 5557   constraint(ALLOC_IN_RC(v15_reg));
 5558   match(RegD);
 5559   op_cost(0);
 5560   format %{ %}
 5561   interface(REG_INTER);
 5562 %}
 5563 
 5564 operand vRegD_V16()
 5565 %{
 5566   constraint(ALLOC_IN_RC(v16_reg));
 5567   match(RegD);
 5568   op_cost(0);
 5569   format %{ %}
 5570   interface(REG_INTER);
 5571 %}
 5572 
 5573 operand vRegD_V17()
 5574 %{
 5575   constraint(ALLOC_IN_RC(v17_reg));
 5576   match(RegD);
 5577   op_cost(0);
 5578   format %{ %}
 5579   interface(REG_INTER);
 5580 %}
 5581 
 5582 operand vRegD_V18()
 5583 %{
 5584   constraint(ALLOC_IN_RC(v18_reg));
 5585   match(RegD);
 5586   op_cost(0);
 5587   format %{ %}
 5588   interface(REG_INTER);
 5589 %}
 5590 
 5591 operand vRegD_V19()
 5592 %{
 5593   constraint(ALLOC_IN_RC(v19_reg));
 5594   match(RegD);
 5595   op_cost(0);
 5596   format %{ %}
 5597   interface(REG_INTER);
 5598 %}
 5599 
 5600 operand vRegD_V20()
 5601 %{
 5602   constraint(ALLOC_IN_RC(v20_reg));
 5603   match(RegD);
 5604   op_cost(0);
 5605   format %{ %}
 5606   interface(REG_INTER);
 5607 %}
 5608 
 5609 operand vRegD_V21()
 5610 %{
 5611   constraint(ALLOC_IN_RC(v21_reg));
 5612   match(RegD);
 5613   op_cost(0);
 5614   format %{ %}
 5615   interface(REG_INTER);
 5616 %}
 5617 
 5618 operand vRegD_V22()
 5619 %{
 5620   constraint(ALLOC_IN_RC(v22_reg));
 5621   match(RegD);
 5622   op_cost(0);
 5623   format %{ %}
 5624   interface(REG_INTER);
 5625 %}
 5626 
 5627 operand vRegD_V23()
 5628 %{
 5629   constraint(ALLOC_IN_RC(v23_reg));
 5630   match(RegD);
 5631   op_cost(0);
 5632   format %{ %}
 5633   interface(REG_INTER);
 5634 %}
 5635 
 5636 operand vRegD_V24()
 5637 %{
 5638   constraint(ALLOC_IN_RC(v24_reg));
 5639   match(RegD);
 5640   op_cost(0);
 5641   format %{ %}
 5642   interface(REG_INTER);
 5643 %}
 5644 
 5645 operand vRegD_V25()
 5646 %{
 5647   constraint(ALLOC_IN_RC(v25_reg));
 5648   match(RegD);
 5649   op_cost(0);
 5650   format %{ %}
 5651   interface(REG_INTER);
 5652 %}
 5653 
 5654 operand vRegD_V26()
 5655 %{
 5656   constraint(ALLOC_IN_RC(v26_reg));
 5657   match(RegD);
 5658   op_cost(0);
 5659   format %{ %}
 5660   interface(REG_INTER);
 5661 %}
 5662 
 5663 operand vRegD_V27()
 5664 %{
 5665   constraint(ALLOC_IN_RC(v27_reg));
 5666   match(RegD);
 5667   op_cost(0);
 5668   format %{ %}
 5669   interface(REG_INTER);
 5670 %}
 5671 
 5672 operand vRegD_V28()
 5673 %{
 5674   constraint(ALLOC_IN_RC(v28_reg));
 5675   match(RegD);
 5676   op_cost(0);
 5677   format %{ %}
 5678   interface(REG_INTER);
 5679 %}
 5680 
 5681 operand vRegD_V29()
 5682 %{
 5683   constraint(ALLOC_IN_RC(v29_reg));
 5684   match(RegD);
 5685   op_cost(0);
 5686   format %{ %}
 5687   interface(REG_INTER);
 5688 %}
 5689 
 5690 operand vRegD_V30()
 5691 %{
 5692   constraint(ALLOC_IN_RC(v30_reg));
 5693   match(RegD);
 5694   op_cost(0);
 5695   format %{ %}
 5696   interface(REG_INTER);
 5697 %}
 5698 
 5699 operand vRegD_V31()
 5700 %{
 5701   constraint(ALLOC_IN_RC(v31_reg));
 5702   match(RegD);
 5703   op_cost(0);
 5704   format %{ %}
 5705   interface(REG_INTER);
 5706 %}
 5707 
 5708 operand pReg()
 5709 %{
 5710   constraint(ALLOC_IN_RC(pr_reg));
 5711   match(RegVectMask);
 5712   match(pRegGov);
 5713   op_cost(0);
 5714   format %{ %}
 5715   interface(REG_INTER);
 5716 %}
 5717 
 5718 operand pRegGov()
 5719 %{
 5720   constraint(ALLOC_IN_RC(gov_pr));
 5721   match(RegVectMask);
 5722   op_cost(0);
 5723   format %{ %}
 5724   interface(REG_INTER);
 5725 %}
 5726 
 5727 // Flags register, used as output of signed compare instructions
 5728 
 5729 // note that on AArch64 we also use this register as the output for
 5730 // for floating point compare instructions (CmpF CmpD). this ensures
 5731 // that ordered inequality tests use GT, GE, LT or LE none of which
 5732 // pass through cases where the result is unordered i.e. one or both
 5733 // inputs to the compare is a NaN. this means that the ideal code can
 5734 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5735 // (where the comparison should always fail). EQ and NE tests are
 5736 // always generated in ideal code so that unordered folds into the NE
 5737 // case, matching the behaviour of AArch64 NE.
 5738 //
 5739 // This differs from x86 where the outputs of FP compares use a
 5740 // special FP flags registers and where compares based on this
 5741 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5742 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5743 // to explicitly handle the unordered case in branches. x86 also has
 5744 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5745 
 5746 operand rFlagsReg()
 5747 %{
 5748   constraint(ALLOC_IN_RC(int_flags));
 5749   match(RegFlags);
 5750 
 5751   op_cost(0);
 5752   format %{ "RFLAGS" %}
 5753   interface(REG_INTER);
 5754 %}
 5755 
 5756 // Flags register, used as output of unsigned compare instructions
 5757 operand rFlagsRegU()
 5758 %{
 5759   constraint(ALLOC_IN_RC(int_flags));
 5760   match(RegFlags);
 5761 
 5762   op_cost(0);
 5763   format %{ "RFLAGSU" %}
 5764   interface(REG_INTER);
 5765 %}
 5766 
 5767 // Special Registers
 5768 
 5769 // Method Register
 5770 operand inline_cache_RegP(iRegP reg)
 5771 %{
 5772   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5773   match(reg);
 5774   match(iRegPNoSp);
 5775   op_cost(0);
 5776   format %{ %}
 5777   interface(REG_INTER);
 5778 %}
 5779 
 5780 // Thread Register
 5781 operand thread_RegP(iRegP reg)
 5782 %{
 5783   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5784   match(reg);
 5785   op_cost(0);
 5786   format %{ %}
 5787   interface(REG_INTER);
 5788 %}
 5789 
 5790 operand lr_RegP(iRegP reg)
 5791 %{
 5792   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5793   match(reg);
 5794   op_cost(0);
 5795   format %{ %}
 5796   interface(REG_INTER);
 5797 %}
 5798 
 5799 //----------Memory Operands----------------------------------------------------
 5800 
 5801 operand indirect(iRegP reg)
 5802 %{
 5803   constraint(ALLOC_IN_RC(ptr_reg));
 5804   match(reg);
 5805   op_cost(0);
 5806   format %{ "[$reg]" %}
 5807   interface(MEMORY_INTER) %{
 5808     base($reg);
 5809     index(0xffffffff);
 5810     scale(0x0);
 5811     disp(0x0);
 5812   %}
 5813 %}
 5814 
 5815 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5816 %{
 5817   constraint(ALLOC_IN_RC(ptr_reg));
 5818   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5819   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5820   op_cost(0);
 5821   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5822   interface(MEMORY_INTER) %{
 5823     base($reg);
 5824     index($ireg);
 5825     scale($scale);
 5826     disp(0x0);
 5827   %}
 5828 %}
 5829 
 5830 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5831 %{
 5832   constraint(ALLOC_IN_RC(ptr_reg));
 5833   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5834   match(AddP reg (LShiftL lreg scale));
 5835   op_cost(0);
 5836   format %{ "$reg, $lreg lsl($scale)" %}
 5837   interface(MEMORY_INTER) %{
 5838     base($reg);
 5839     index($lreg);
 5840     scale($scale);
 5841     disp(0x0);
 5842   %}
 5843 %}
 5844 
 5845 operand indIndexI2L(iRegP reg, iRegI ireg)
 5846 %{
 5847   constraint(ALLOC_IN_RC(ptr_reg));
 5848   match(AddP reg (ConvI2L ireg));
 5849   op_cost(0);
 5850   format %{ "$reg, $ireg, 0, I2L" %}
 5851   interface(MEMORY_INTER) %{
 5852     base($reg);
 5853     index($ireg);
 5854     scale(0x0);
 5855     disp(0x0);
 5856   %}
 5857 %}
 5858 
 5859 operand indIndex(iRegP reg, iRegL lreg)
 5860 %{
 5861   constraint(ALLOC_IN_RC(ptr_reg));
 5862   match(AddP reg lreg);
 5863   op_cost(0);
 5864   format %{ "$reg, $lreg" %}
 5865   interface(MEMORY_INTER) %{
 5866     base($reg);
 5867     index($lreg);
 5868     scale(0x0);
 5869     disp(0x0);
 5870   %}
 5871 %}
 5872 
 5873 operand indOffI(iRegP reg, immIOffset off)
 5874 %{
 5875   constraint(ALLOC_IN_RC(ptr_reg));
 5876   match(AddP reg off);
 5877   op_cost(0);
 5878   format %{ "[$reg, $off]" %}
 5879   interface(MEMORY_INTER) %{
 5880     base($reg);
 5881     index(0xffffffff);
 5882     scale(0x0);
 5883     disp($off);
 5884   %}
 5885 %}
 5886 
 5887 operand indOffI1(iRegP reg, immIOffset1 off)
 5888 %{
 5889   constraint(ALLOC_IN_RC(ptr_reg));
 5890   match(AddP reg off);
 5891   op_cost(0);
 5892   format %{ "[$reg, $off]" %}
 5893   interface(MEMORY_INTER) %{
 5894     base($reg);
 5895     index(0xffffffff);
 5896     scale(0x0);
 5897     disp($off);
 5898   %}
 5899 %}
 5900 
 5901 operand indOffI2(iRegP reg, immIOffset2 off)
 5902 %{
 5903   constraint(ALLOC_IN_RC(ptr_reg));
 5904   match(AddP reg off);
 5905   op_cost(0);
 5906   format %{ "[$reg, $off]" %}
 5907   interface(MEMORY_INTER) %{
 5908     base($reg);
 5909     index(0xffffffff);
 5910     scale(0x0);
 5911     disp($off);
 5912   %}
 5913 %}
 5914 
 5915 operand indOffI4(iRegP reg, immIOffset4 off)
 5916 %{
 5917   constraint(ALLOC_IN_RC(ptr_reg));
 5918   match(AddP reg off);
 5919   op_cost(0);
 5920   format %{ "[$reg, $off]" %}
 5921   interface(MEMORY_INTER) %{
 5922     base($reg);
 5923     index(0xffffffff);
 5924     scale(0x0);
 5925     disp($off);
 5926   %}
 5927 %}
 5928 
 5929 operand indOffI8(iRegP reg, immIOffset8 off)
 5930 %{
 5931   constraint(ALLOC_IN_RC(ptr_reg));
 5932   match(AddP reg off);
 5933   op_cost(0);
 5934   format %{ "[$reg, $off]" %}
 5935   interface(MEMORY_INTER) %{
 5936     base($reg);
 5937     index(0xffffffff);
 5938     scale(0x0);
 5939     disp($off);
 5940   %}
 5941 %}
 5942 
 5943 operand indOffI16(iRegP reg, immIOffset16 off)
 5944 %{
 5945   constraint(ALLOC_IN_RC(ptr_reg));
 5946   match(AddP reg off);
 5947   op_cost(0);
 5948   format %{ "[$reg, $off]" %}
 5949   interface(MEMORY_INTER) %{
 5950     base($reg);
 5951     index(0xffffffff);
 5952     scale(0x0);
 5953     disp($off);
 5954   %}
 5955 %}
 5956 
 5957 operand indOffL(iRegP reg, immLoffset off)
 5958 %{
 5959   constraint(ALLOC_IN_RC(ptr_reg));
 5960   match(AddP reg off);
 5961   op_cost(0);
 5962   format %{ "[$reg, $off]" %}
 5963   interface(MEMORY_INTER) %{
 5964     base($reg);
 5965     index(0xffffffff);
 5966     scale(0x0);
 5967     disp($off);
 5968   %}
 5969 %}
 5970 
 5971 operand indOffL1(iRegP reg, immLoffset1 off)
 5972 %{
 5973   constraint(ALLOC_IN_RC(ptr_reg));
 5974   match(AddP reg off);
 5975   op_cost(0);
 5976   format %{ "[$reg, $off]" %}
 5977   interface(MEMORY_INTER) %{
 5978     base($reg);
 5979     index(0xffffffff);
 5980     scale(0x0);
 5981     disp($off);
 5982   %}
 5983 %}
 5984 
 5985 operand indOffL2(iRegP reg, immLoffset2 off)
 5986 %{
 5987   constraint(ALLOC_IN_RC(ptr_reg));
 5988   match(AddP reg off);
 5989   op_cost(0);
 5990   format %{ "[$reg, $off]" %}
 5991   interface(MEMORY_INTER) %{
 5992     base($reg);
 5993     index(0xffffffff);
 5994     scale(0x0);
 5995     disp($off);
 5996   %}
 5997 %}
 5998 
 5999 operand indOffL4(iRegP reg, immLoffset4 off)
 6000 %{
 6001   constraint(ALLOC_IN_RC(ptr_reg));
 6002   match(AddP reg off);
 6003   op_cost(0);
 6004   format %{ "[$reg, $off]" %}
 6005   interface(MEMORY_INTER) %{
 6006     base($reg);
 6007     index(0xffffffff);
 6008     scale(0x0);
 6009     disp($off);
 6010   %}
 6011 %}
 6012 
 6013 operand indOffL8(iRegP reg, immLoffset8 off)
 6014 %{
 6015   constraint(ALLOC_IN_RC(ptr_reg));
 6016   match(AddP reg off);
 6017   op_cost(0);
 6018   format %{ "[$reg, $off]" %}
 6019   interface(MEMORY_INTER) %{
 6020     base($reg);
 6021     index(0xffffffff);
 6022     scale(0x0);
 6023     disp($off);
 6024   %}
 6025 %}
 6026 
 6027 operand indOffL16(iRegP reg, immLoffset16 off)
 6028 %{
 6029   constraint(ALLOC_IN_RC(ptr_reg));
 6030   match(AddP reg off);
 6031   op_cost(0);
 6032   format %{ "[$reg, $off]" %}
 6033   interface(MEMORY_INTER) %{
 6034     base($reg);
 6035     index(0xffffffff);
 6036     scale(0x0);
 6037     disp($off);
 6038   %}
 6039 %}
 6040 
 6041 operand indirectN(iRegN reg)
 6042 %{
 6043   predicate(CompressedOops::shift() == 0);
 6044   constraint(ALLOC_IN_RC(ptr_reg));
 6045   match(DecodeN reg);
 6046   op_cost(0);
 6047   format %{ "[$reg]\t# narrow" %}
 6048   interface(MEMORY_INTER) %{
 6049     base($reg);
 6050     index(0xffffffff);
 6051     scale(0x0);
 6052     disp(0x0);
 6053   %}
 6054 %}
 6055 
 6056 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 6057 %{
 6058   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6059   constraint(ALLOC_IN_RC(ptr_reg));
 6060   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 6061   op_cost(0);
 6062   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 6063   interface(MEMORY_INTER) %{
 6064     base($reg);
 6065     index($ireg);
 6066     scale($scale);
 6067     disp(0x0);
 6068   %}
 6069 %}
 6070 
 6071 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 6072 %{
 6073   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6074   constraint(ALLOC_IN_RC(ptr_reg));
 6075   match(AddP (DecodeN reg) (LShiftL lreg scale));
 6076   op_cost(0);
 6077   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 6078   interface(MEMORY_INTER) %{
 6079     base($reg);
 6080     index($lreg);
 6081     scale($scale);
 6082     disp(0x0);
 6083   %}
 6084 %}
 6085 
 6086 operand indIndexI2LN(iRegN reg, iRegI ireg)
 6087 %{
 6088   predicate(CompressedOops::shift() == 0);
 6089   constraint(ALLOC_IN_RC(ptr_reg));
 6090   match(AddP (DecodeN reg) (ConvI2L ireg));
 6091   op_cost(0);
 6092   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 6093   interface(MEMORY_INTER) %{
 6094     base($reg);
 6095     index($ireg);
 6096     scale(0x0);
 6097     disp(0x0);
 6098   %}
 6099 %}
 6100 
 6101 operand indIndexN(iRegN reg, iRegL lreg)
 6102 %{
 6103   predicate(CompressedOops::shift() == 0);
 6104   constraint(ALLOC_IN_RC(ptr_reg));
 6105   match(AddP (DecodeN reg) lreg);
 6106   op_cost(0);
 6107   format %{ "$reg, $lreg\t# narrow" %}
 6108   interface(MEMORY_INTER) %{
 6109     base($reg);
 6110     index($lreg);
 6111     scale(0x0);
 6112     disp(0x0);
 6113   %}
 6114 %}
 6115 
 6116 operand indOffIN(iRegN reg, immIOffset off)
 6117 %{
 6118   predicate(CompressedOops::shift() == 0);
 6119   constraint(ALLOC_IN_RC(ptr_reg));
 6120   match(AddP (DecodeN reg) off);
 6121   op_cost(0);
 6122   format %{ "[$reg, $off]\t# narrow" %}
 6123   interface(MEMORY_INTER) %{
 6124     base($reg);
 6125     index(0xffffffff);
 6126     scale(0x0);
 6127     disp($off);
 6128   %}
 6129 %}
 6130 
 6131 operand indOffLN(iRegN reg, immLoffset off)
 6132 %{
 6133   predicate(CompressedOops::shift() == 0);
 6134   constraint(ALLOC_IN_RC(ptr_reg));
 6135   match(AddP (DecodeN reg) off);
 6136   op_cost(0);
 6137   format %{ "[$reg, $off]\t# narrow" %}
 6138   interface(MEMORY_INTER) %{
 6139     base($reg);
 6140     index(0xffffffff);
 6141     scale(0x0);
 6142     disp($off);
 6143   %}
 6144 %}
 6145 
 6146 
 6147 
 6148 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6149 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6150 %{
 6151   constraint(ALLOC_IN_RC(ptr_reg));
 6152   match(AddP reg off);
 6153   op_cost(0);
 6154   format %{ "[$reg, $off]" %}
 6155   interface(MEMORY_INTER) %{
 6156     base($reg);
 6157     index(0xffffffff);
 6158     scale(0x0);
 6159     disp($off);
 6160   %}
 6161 %}
 6162 
 6163 //----------Special Memory Operands--------------------------------------------
 6164 // Stack Slot Operand - This operand is used for loading and storing temporary
 6165 //                      values on the stack where a match requires a value to
 6166 //                      flow through memory.
 6167 operand stackSlotP(sRegP reg)
 6168 %{
 6169   constraint(ALLOC_IN_RC(stack_slots));
 6170   op_cost(100);
 6171   // No match rule because this operand is only generated in matching
 6172   // match(RegP);
 6173   format %{ "[$reg]" %}
 6174   interface(MEMORY_INTER) %{
 6175     base(0x1e);  // RSP
 6176     index(0x0);  // No Index
 6177     scale(0x0);  // No Scale
 6178     disp($reg);  // Stack Offset
 6179   %}
 6180 %}
 6181 
 6182 operand stackSlotI(sRegI reg)
 6183 %{
 6184   constraint(ALLOC_IN_RC(stack_slots));
 6185   // No match rule because this operand is only generated in matching
 6186   // match(RegI);
 6187   format %{ "[$reg]" %}
 6188   interface(MEMORY_INTER) %{
 6189     base(0x1e);  // RSP
 6190     index(0x0);  // No Index
 6191     scale(0x0);  // No Scale
 6192     disp($reg);  // Stack Offset
 6193   %}
 6194 %}
 6195 
 6196 operand stackSlotF(sRegF reg)
 6197 %{
 6198   constraint(ALLOC_IN_RC(stack_slots));
 6199   // No match rule because this operand is only generated in matching
 6200   // match(RegF);
 6201   format %{ "[$reg]" %}
 6202   interface(MEMORY_INTER) %{
 6203     base(0x1e);  // RSP
 6204     index(0x0);  // No Index
 6205     scale(0x0);  // No Scale
 6206     disp($reg);  // Stack Offset
 6207   %}
 6208 %}
 6209 
 6210 operand stackSlotD(sRegD reg)
 6211 %{
 6212   constraint(ALLOC_IN_RC(stack_slots));
 6213   // No match rule because this operand is only generated in matching
 6214   // match(RegD);
 6215   format %{ "[$reg]" %}
 6216   interface(MEMORY_INTER) %{
 6217     base(0x1e);  // RSP
 6218     index(0x0);  // No Index
 6219     scale(0x0);  // No Scale
 6220     disp($reg);  // Stack Offset
 6221   %}
 6222 %}
 6223 
 6224 operand stackSlotL(sRegL reg)
 6225 %{
 6226   constraint(ALLOC_IN_RC(stack_slots));
 6227   // No match rule because this operand is only generated in matching
 6228   // match(RegL);
 6229   format %{ "[$reg]" %}
 6230   interface(MEMORY_INTER) %{
 6231     base(0x1e);  // RSP
 6232     index(0x0);  // No Index
 6233     scale(0x0);  // No Scale
 6234     disp($reg);  // Stack Offset
 6235   %}
 6236 %}
 6237 
 6238 // Operands for expressing Control Flow
 6239 // NOTE: Label is a predefined operand which should not be redefined in
 6240 //       the AD file. It is generically handled within the ADLC.
 6241 
 6242 //----------Conditional Branch Operands----------------------------------------
 6243 // Comparison Op  - This is the operation of the comparison, and is limited to
 6244 //                  the following set of codes:
 6245 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6246 //
 6247 // Other attributes of the comparison, such as unsignedness, are specified
 6248 // by the comparison instruction that sets a condition code flags register.
 6249 // That result is represented by a flags operand whose subtype is appropriate
 6250 // to the unsignedness (etc.) of the comparison.
 6251 //
 6252 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6253 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6254 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6255 
 6256 // used for signed integral comparisons and fp comparisons
 6257 
 6258 operand cmpOp()
 6259 %{
 6260   match(Bool);
 6261 
 6262   format %{ "" %}
 6263   interface(COND_INTER) %{
 6264     equal(0x0, "eq");
 6265     not_equal(0x1, "ne");
 6266     less(0xb, "lt");
 6267     greater_equal(0xa, "ge");
 6268     less_equal(0xd, "le");
 6269     greater(0xc, "gt");
 6270     overflow(0x6, "vs");
 6271     no_overflow(0x7, "vc");
 6272   %}
 6273 %}
 6274 
 6275 // used for unsigned integral comparisons
 6276 
 6277 operand cmpOpU()
 6278 %{
 6279   match(Bool);
 6280 
 6281   format %{ "" %}
 6282   interface(COND_INTER) %{
 6283     equal(0x0, "eq");
 6284     not_equal(0x1, "ne");
 6285     less(0x3, "lo");
 6286     greater_equal(0x2, "hs");
 6287     less_equal(0x9, "ls");
 6288     greater(0x8, "hi");
 6289     overflow(0x6, "vs");
 6290     no_overflow(0x7, "vc");
 6291   %}
 6292 %}
 6293 
 6294 // used for certain integral comparisons which can be
 6295 // converted to cbxx or tbxx instructions
 6296 
 6297 operand cmpOpEqNe()
 6298 %{
 6299   match(Bool);
 6300   op_cost(0);
 6301   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6302             || n->as_Bool()->_test._test == BoolTest::eq);
 6303 
 6304   format %{ "" %}
 6305   interface(COND_INTER) %{
 6306     equal(0x0, "eq");
 6307     not_equal(0x1, "ne");
 6308     less(0xb, "lt");
 6309     greater_equal(0xa, "ge");
 6310     less_equal(0xd, "le");
 6311     greater(0xc, "gt");
 6312     overflow(0x6, "vs");
 6313     no_overflow(0x7, "vc");
 6314   %}
 6315 %}
 6316 
 6317 // used for certain integral comparisons which can be
 6318 // converted to cbxx or tbxx instructions
 6319 
 6320 operand cmpOpLtGe()
 6321 %{
 6322   match(Bool);
 6323   op_cost(0);
 6324 
 6325   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6326             || n->as_Bool()->_test._test == BoolTest::ge);
 6327 
 6328   format %{ "" %}
 6329   interface(COND_INTER) %{
 6330     equal(0x0, "eq");
 6331     not_equal(0x1, "ne");
 6332     less(0xb, "lt");
 6333     greater_equal(0xa, "ge");
 6334     less_equal(0xd, "le");
 6335     greater(0xc, "gt");
 6336     overflow(0x6, "vs");
 6337     no_overflow(0x7, "vc");
 6338   %}
 6339 %}
 6340 
 6341 // used for certain unsigned integral comparisons which can be
 6342 // converted to cbxx or tbxx instructions
 6343 
 6344 operand cmpOpUEqNeLtGe()
 6345 %{
 6346   match(Bool);
 6347   op_cost(0);
 6348 
 6349   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6350             || n->as_Bool()->_test._test == BoolTest::ne
 6351             || n->as_Bool()->_test._test == BoolTest::lt
 6352             || n->as_Bool()->_test._test == BoolTest::ge);
 6353 
 6354   format %{ "" %}
 6355   interface(COND_INTER) %{
 6356     equal(0x0, "eq");
 6357     not_equal(0x1, "ne");
 6358     less(0xb, "lt");
 6359     greater_equal(0xa, "ge");
 6360     less_equal(0xd, "le");
 6361     greater(0xc, "gt");
 6362     overflow(0x6, "vs");
 6363     no_overflow(0x7, "vc");
 6364   %}
 6365 %}
 6366 
 6367 // Special operand allowing long args to int ops to be truncated for free
 6368 
 6369 operand iRegL2I(iRegL reg) %{
 6370 
 6371   op_cost(0);
 6372 
 6373   match(ConvL2I reg);
 6374 
 6375   format %{ "l2i($reg)" %}
 6376 
 6377   interface(REG_INTER)
 6378 %}
 6379 
 6380 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6381 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6382 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6383 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6384 
 6385 //----------OPERAND CLASSES----------------------------------------------------
 6386 // Operand Classes are groups of operands that are used as to simplify
 6387 // instruction definitions by not requiring the AD writer to specify
 6388 // separate instructions for every form of operand when the
 6389 // instruction accepts multiple operand types with the same basic
 6390 // encoding and format. The classic case of this is memory operands.
 6391 
 6392 // memory is used to define read/write location for load/store
 6393 // instruction defs. we can turn a memory op into an Address
 6394 
 6395 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6396                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6397 
 6398 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6399                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6400 
 6401 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6402                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6403 
 6404 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6405                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6406 
 6407 // All of the memory operands. For the pipeline description.
 6408 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6409                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6410                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6411 
 6412 
 6413 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6414 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6415 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6416 // can be elided because the 32-bit instruction will just employ the
 6417 // lower 32 bits anyway.
 6418 //
 6419 // n.b. this does not elide all L2I conversions. if the truncated
 6420 // value is consumed by more than one operation then the ConvL2I
 6421 // cannot be bundled into the consuming nodes so an l2i gets planted
 6422 // (actually a movw $dst $src) and the downstream instructions consume
 6423 // the result of the l2i as an iRegI input. That's a shame since the
 6424 // movw is actually redundant but its not too costly.
 6425 
 6426 opclass iRegIorL2I(iRegI, iRegL2I);
 6427 
 6428 //----------PIPELINE-----------------------------------------------------------
 6429 // Rules which define the behavior of the target architectures pipeline.
 6430 
 6431 // For specific pipelines, eg A53, define the stages of that pipeline
 6432 //pipe_desc(ISS, EX1, EX2, WR);
 6433 #define ISS S0
 6434 #define EX1 S1
 6435 #define EX2 S2
 6436 #define WR  S3
 6437 
 6438 // Integer ALU reg operation
 6439 pipeline %{
 6440 
 6441 attributes %{
 6442   // ARM instructions are of fixed length
 6443   fixed_size_instructions;        // Fixed size instructions TODO does
 6444   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6445   // ARM instructions come in 32-bit word units
 6446   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6447   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6448   instruction_fetch_units = 1;       // of 64 bytes
 6449 
 6450   // List of nop instructions
 6451   nops( MachNop );
 6452 %}
 6453 
 6454 // We don't use an actual pipeline model so don't care about resources
 6455 // or description. we do use pipeline classes to introduce fixed
 6456 // latencies
 6457 
 6458 //----------RESOURCES----------------------------------------------------------
 6459 // Resources are the functional units available to the machine
 6460 
 6461 resources( INS0, INS1, INS01 = INS0 | INS1,
 6462            ALU0, ALU1, ALU = ALU0 | ALU1,
 6463            MAC,
 6464            DIV,
 6465            BRANCH,
 6466            LDST,
 6467            NEON_FP);
 6468 
 6469 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6470 // Pipeline Description specifies the stages in the machine's pipeline
 6471 
 6472 // Define the pipeline as a generic 6 stage pipeline
 6473 pipe_desc(S0, S1, S2, S3, S4, S5);
 6474 
 6475 //----------PIPELINE CLASSES---------------------------------------------------
 6476 // Pipeline Classes describe the stages in which input and output are
 6477 // referenced by the hardware pipeline.
 6478 
 6479 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6480 %{
 6481   single_instruction;
 6482   src1   : S1(read);
 6483   src2   : S2(read);
 6484   dst    : S5(write);
 6485   INS01  : ISS;
 6486   NEON_FP : S5;
 6487 %}
 6488 
 6489 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6490 %{
 6491   single_instruction;
 6492   src1   : S1(read);
 6493   src2   : S2(read);
 6494   dst    : S5(write);
 6495   INS01  : ISS;
 6496   NEON_FP : S5;
 6497 %}
 6498 
 6499 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6500 %{
 6501   single_instruction;
 6502   src    : S1(read);
 6503   dst    : S5(write);
 6504   INS01  : ISS;
 6505   NEON_FP : S5;
 6506 %}
 6507 
 6508 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6509 %{
 6510   single_instruction;
 6511   src    : S1(read);
 6512   dst    : S5(write);
 6513   INS01  : ISS;
 6514   NEON_FP : S5;
 6515 %}
 6516 
 6517 pipe_class fp_d2f(vRegF dst, vRegD src)
 6518 %{
 6519   single_instruction;
 6520   src    : S1(read);
 6521   dst    : S5(write);
 6522   INS01  : ISS;
 6523   NEON_FP : S5;
 6524 %}
 6525 
 6526 pipe_class fp_f2d(vRegD dst, vRegF src)
 6527 %{
 6528   single_instruction;
 6529   src    : S1(read);
 6530   dst    : S5(write);
 6531   INS01  : ISS;
 6532   NEON_FP : S5;
 6533 %}
 6534 
 6535 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6536 %{
 6537   single_instruction;
 6538   src    : S1(read);
 6539   dst    : S5(write);
 6540   INS01  : ISS;
 6541   NEON_FP : S5;
 6542 %}
 6543 
 6544 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6545 %{
 6546   single_instruction;
 6547   src    : S1(read);
 6548   dst    : S5(write);
 6549   INS01  : ISS;
 6550   NEON_FP : S5;
 6551 %}
 6552 
 6553 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6554 %{
 6555   single_instruction;
 6556   src    : S1(read);
 6557   dst    : S5(write);
 6558   INS01  : ISS;
 6559   NEON_FP : S5;
 6560 %}
 6561 
 6562 pipe_class fp_l2f(vRegF dst, iRegL src)
 6563 %{
 6564   single_instruction;
 6565   src    : S1(read);
 6566   dst    : S5(write);
 6567   INS01  : ISS;
 6568   NEON_FP : S5;
 6569 %}
 6570 
 6571 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6572 %{
 6573   single_instruction;
 6574   src    : S1(read);
 6575   dst    : S5(write);
 6576   INS01  : ISS;
 6577   NEON_FP : S5;
 6578 %}
 6579 
 6580 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6581 %{
 6582   single_instruction;
 6583   src    : S1(read);
 6584   dst    : S5(write);
 6585   INS01  : ISS;
 6586   NEON_FP : S5;
 6587 %}
 6588 
 6589 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6590 %{
 6591   single_instruction;
 6592   src    : S1(read);
 6593   dst    : S5(write);
 6594   INS01  : ISS;
 6595   NEON_FP : S5;
 6596 %}
 6597 
 6598 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6599 %{
 6600   single_instruction;
 6601   src    : S1(read);
 6602   dst    : S5(write);
 6603   INS01  : ISS;
 6604   NEON_FP : S5;
 6605 %}
 6606 
 6607 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6608 %{
 6609   single_instruction;
 6610   src1   : S1(read);
 6611   src2   : S2(read);
 6612   dst    : S5(write);
 6613   INS0   : ISS;
 6614   NEON_FP : S5;
 6615 %}
 6616 
 6617 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6618 %{
 6619   single_instruction;
 6620   src1   : S1(read);
 6621   src2   : S2(read);
 6622   dst    : S5(write);
 6623   INS0   : ISS;
 6624   NEON_FP : S5;
 6625 %}
 6626 
 6627 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6628 %{
 6629   single_instruction;
 6630   cr     : S1(read);
 6631   src1   : S1(read);
 6632   src2   : S1(read);
 6633   dst    : S3(write);
 6634   INS01  : ISS;
 6635   NEON_FP : S3;
 6636 %}
 6637 
 6638 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6639 %{
 6640   single_instruction;
 6641   cr     : S1(read);
 6642   src1   : S1(read);
 6643   src2   : S1(read);
 6644   dst    : S3(write);
 6645   INS01  : ISS;
 6646   NEON_FP : S3;
 6647 %}
 6648 
 6649 pipe_class fp_imm_s(vRegF dst)
 6650 %{
 6651   single_instruction;
 6652   dst    : S3(write);
 6653   INS01  : ISS;
 6654   NEON_FP : S3;
 6655 %}
 6656 
 6657 pipe_class fp_imm_d(vRegD dst)
 6658 %{
 6659   single_instruction;
 6660   dst    : S3(write);
 6661   INS01  : ISS;
 6662   NEON_FP : S3;
 6663 %}
 6664 
 6665 pipe_class fp_load_constant_s(vRegF dst)
 6666 %{
 6667   single_instruction;
 6668   dst    : S4(write);
 6669   INS01  : ISS;
 6670   NEON_FP : S4;
 6671 %}
 6672 
 6673 pipe_class fp_load_constant_d(vRegD dst)
 6674 %{
 6675   single_instruction;
 6676   dst    : S4(write);
 6677   INS01  : ISS;
 6678   NEON_FP : S4;
 6679 %}
 6680 
 6681 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6682 %{
 6683   single_instruction;
 6684   dst    : S5(write);
 6685   src1   : S1(read);
 6686   src2   : S1(read);
 6687   INS01  : ISS;
 6688   NEON_FP : S5;
 6689 %}
 6690 
 6691 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6692 %{
 6693   single_instruction;
 6694   dst    : S5(write);
 6695   src1   : S1(read);
 6696   src2   : S1(read);
 6697   INS0   : ISS;
 6698   NEON_FP : S5;
 6699 %}
 6700 
 6701 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6702 %{
 6703   single_instruction;
 6704   dst    : S5(write);
 6705   src1   : S1(read);
 6706   src2   : S1(read);
 6707   dst    : S1(read);
 6708   INS01  : ISS;
 6709   NEON_FP : S5;
 6710 %}
 6711 
 6712 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6713 %{
 6714   single_instruction;
 6715   dst    : S5(write);
 6716   src1   : S1(read);
 6717   src2   : S1(read);
 6718   dst    : S1(read);
 6719   INS0   : ISS;
 6720   NEON_FP : S5;
 6721 %}
 6722 
 6723 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6724 %{
 6725   single_instruction;
 6726   dst    : S4(write);
 6727   src1   : S2(read);
 6728   src2   : S2(read);
 6729   INS01  : ISS;
 6730   NEON_FP : S4;
 6731 %}
 6732 
 6733 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6734 %{
 6735   single_instruction;
 6736   dst    : S4(write);
 6737   src1   : S2(read);
 6738   src2   : S2(read);
 6739   INS0   : ISS;
 6740   NEON_FP : S4;
 6741 %}
 6742 
 6743 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6744 %{
 6745   single_instruction;
 6746   dst    : S3(write);
 6747   src1   : S2(read);
 6748   src2   : S2(read);
 6749   INS01  : ISS;
 6750   NEON_FP : S3;
 6751 %}
 6752 
 6753 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6754 %{
 6755   single_instruction;
 6756   dst    : S3(write);
 6757   src1   : S2(read);
 6758   src2   : S2(read);
 6759   INS0   : ISS;
 6760   NEON_FP : S3;
 6761 %}
 6762 
 6763 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6764 %{
 6765   single_instruction;
 6766   dst    : S3(write);
 6767   src    : S1(read);
 6768   shift  : S1(read);
 6769   INS01  : ISS;
 6770   NEON_FP : S3;
 6771 %}
 6772 
 6773 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6774 %{
 6775   single_instruction;
 6776   dst    : S3(write);
 6777   src    : S1(read);
 6778   shift  : S1(read);
 6779   INS0   : ISS;
 6780   NEON_FP : S3;
 6781 %}
 6782 
 6783 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6784 %{
 6785   single_instruction;
 6786   dst    : S3(write);
 6787   src    : S1(read);
 6788   INS01  : ISS;
 6789   NEON_FP : S3;
 6790 %}
 6791 
 6792 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6793 %{
 6794   single_instruction;
 6795   dst    : S3(write);
 6796   src    : S1(read);
 6797   INS0   : ISS;
 6798   NEON_FP : S3;
 6799 %}
 6800 
 6801 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6802 %{
 6803   single_instruction;
 6804   dst    : S5(write);
 6805   src1   : S1(read);
 6806   src2   : S1(read);
 6807   INS01  : ISS;
 6808   NEON_FP : S5;
 6809 %}
 6810 
 6811 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6812 %{
 6813   single_instruction;
 6814   dst    : S5(write);
 6815   src1   : S1(read);
 6816   src2   : S1(read);
 6817   INS0   : ISS;
 6818   NEON_FP : S5;
 6819 %}
 6820 
 6821 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6822 %{
 6823   single_instruction;
 6824   dst    : S5(write);
 6825   src1   : S1(read);
 6826   src2   : S1(read);
 6827   INS0   : ISS;
 6828   NEON_FP : S5;
 6829 %}
 6830 
 6831 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6832 %{
 6833   single_instruction;
 6834   dst    : S5(write);
 6835   src1   : S1(read);
 6836   src2   : S1(read);
 6837   INS0   : ISS;
 6838   NEON_FP : S5;
 6839 %}
 6840 
 6841 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6842 %{
 6843   single_instruction;
 6844   dst    : S5(write);
 6845   src    : S1(read);
 6846   INS0   : ISS;
 6847   NEON_FP : S5;
 6848 %}
 6849 
 6850 pipe_class vunop_fp64(vecD dst, vecD src)
 6851 %{
 6852   single_instruction;
 6853   dst    : S5(write);
 6854   src    : S1(read);
 6855   INS01  : ISS;
 6856   NEON_FP : S5;
 6857 %}
 6858 
 6859 pipe_class vunop_fp128(vecX dst, vecX src)
 6860 %{
 6861   single_instruction;
 6862   dst    : S5(write);
 6863   src    : S1(read);
 6864   INS0   : ISS;
 6865   NEON_FP : S5;
 6866 %}
 6867 
 6868 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6869 %{
 6870   single_instruction;
 6871   dst    : S3(write);
 6872   src    : S1(read);
 6873   INS01  : ISS;
 6874   NEON_FP : S3;
 6875 %}
 6876 
 6877 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6878 %{
 6879   single_instruction;
 6880   dst    : S3(write);
 6881   src    : S1(read);
 6882   INS01  : ISS;
 6883   NEON_FP : S3;
 6884 %}
 6885 
 6886 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6887 %{
 6888   single_instruction;
 6889   dst    : S3(write);
 6890   src    : S1(read);
 6891   INS01  : ISS;
 6892   NEON_FP : S3;
 6893 %}
 6894 
 6895 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6896 %{
 6897   single_instruction;
 6898   dst    : S3(write);
 6899   src    : S1(read);
 6900   INS01  : ISS;
 6901   NEON_FP : S3;
 6902 %}
 6903 
 6904 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6905 %{
 6906   single_instruction;
 6907   dst    : S3(write);
 6908   src    : S1(read);
 6909   INS01  : ISS;
 6910   NEON_FP : S3;
 6911 %}
 6912 
 6913 pipe_class vmovi_reg_imm64(vecD dst)
 6914 %{
 6915   single_instruction;
 6916   dst    : S3(write);
 6917   INS01  : ISS;
 6918   NEON_FP : S3;
 6919 %}
 6920 
 6921 pipe_class vmovi_reg_imm128(vecX dst)
 6922 %{
 6923   single_instruction;
 6924   dst    : S3(write);
 6925   INS0   : ISS;
 6926   NEON_FP : S3;
 6927 %}
 6928 
 6929 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6930 %{
 6931   single_instruction;
 6932   dst    : S5(write);
 6933   mem    : ISS(read);
 6934   INS01  : ISS;
 6935   NEON_FP : S3;
 6936 %}
 6937 
 6938 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6939 %{
 6940   single_instruction;
 6941   dst    : S5(write);
 6942   mem    : ISS(read);
 6943   INS01  : ISS;
 6944   NEON_FP : S3;
 6945 %}
 6946 
 6947 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6948 %{
 6949   single_instruction;
 6950   mem    : ISS(read);
 6951   src    : S2(read);
 6952   INS01  : ISS;
 6953   NEON_FP : S3;
 6954 %}
 6955 
 6956 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6957 %{
 6958   single_instruction;
 6959   mem    : ISS(read);
 6960   src    : S2(read);
 6961   INS01  : ISS;
 6962   NEON_FP : S3;
 6963 %}
 6964 
 6965 //------- Integer ALU operations --------------------------
 6966 
 6967 // Integer ALU reg-reg operation
 6968 // Operands needed in EX1, result generated in EX2
 6969 // Eg.  ADD     x0, x1, x2
 6970 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6971 %{
 6972   single_instruction;
 6973   dst    : EX2(write);
 6974   src1   : EX1(read);
 6975   src2   : EX1(read);
 6976   INS01  : ISS; // Dual issue as instruction 0 or 1
 6977   ALU    : EX2;
 6978 %}
 6979 
 6980 // Integer ALU reg-reg operation with constant shift
 6981 // Shifted register must be available in LATE_ISS instead of EX1
 6982 // Eg.  ADD     x0, x1, x2, LSL #2
 6983 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6984 %{
 6985   single_instruction;
 6986   dst    : EX2(write);
 6987   src1   : EX1(read);
 6988   src2   : ISS(read);
 6989   INS01  : ISS;
 6990   ALU    : EX2;
 6991 %}
 6992 
 6993 // Integer ALU reg operation with constant shift
 6994 // Eg.  LSL     x0, x1, #shift
 6995 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6996 %{
 6997   single_instruction;
 6998   dst    : EX2(write);
 6999   src1   : ISS(read);
 7000   INS01  : ISS;
 7001   ALU    : EX2;
 7002 %}
 7003 
 7004 // Integer ALU reg-reg operation with variable shift
 7005 // Both operands must be available in LATE_ISS instead of EX1
 7006 // Result is available in EX1 instead of EX2
 7007 // Eg.  LSLV    x0, x1, x2
 7008 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 7009 %{
 7010   single_instruction;
 7011   dst    : EX1(write);
 7012   src1   : ISS(read);
 7013   src2   : ISS(read);
 7014   INS01  : ISS;
 7015   ALU    : EX1;
 7016 %}
 7017 
 7018 // Integer ALU reg-reg operation with extract
 7019 // As for _vshift above, but result generated in EX2
 7020 // Eg.  EXTR    x0, x1, x2, #N
 7021 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 7022 %{
 7023   single_instruction;
 7024   dst    : EX2(write);
 7025   src1   : ISS(read);
 7026   src2   : ISS(read);
 7027   INS1   : ISS; // Can only dual issue as Instruction 1
 7028   ALU    : EX1;
 7029 %}
 7030 
 7031 // Integer ALU reg operation
 7032 // Eg.  NEG     x0, x1
 7033 pipe_class ialu_reg(iRegI dst, iRegI src)
 7034 %{
 7035   single_instruction;
 7036   dst    : EX2(write);
 7037   src    : EX1(read);
 7038   INS01  : ISS;
 7039   ALU    : EX2;
 7040 %}
 7041 
 7042 // Integer ALU reg mmediate operation
 7043 // Eg.  ADD     x0, x1, #N
 7044 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 7045 %{
 7046   single_instruction;
 7047   dst    : EX2(write);
 7048   src1   : EX1(read);
 7049   INS01  : ISS;
 7050   ALU    : EX2;
 7051 %}
 7052 
 7053 // Integer ALU immediate operation (no source operands)
 7054 // Eg.  MOV     x0, #N
 7055 pipe_class ialu_imm(iRegI dst)
 7056 %{
 7057   single_instruction;
 7058   dst    : EX1(write);
 7059   INS01  : ISS;
 7060   ALU    : EX1;
 7061 %}
 7062 
 7063 //------- Compare operation -------------------------------
 7064 
 7065 // Compare reg-reg
 7066 // Eg.  CMP     x0, x1
 7067 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 7068 %{
 7069   single_instruction;
 7070 //  fixed_latency(16);
 7071   cr     : EX2(write);
 7072   op1    : EX1(read);
 7073   op2    : EX1(read);
 7074   INS01  : ISS;
 7075   ALU    : EX2;
 7076 %}
 7077 
 7078 // Compare reg-reg
 7079 // Eg.  CMP     x0, #N
 7080 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 7081 %{
 7082   single_instruction;
 7083 //  fixed_latency(16);
 7084   cr     : EX2(write);
 7085   op1    : EX1(read);
 7086   INS01  : ISS;
 7087   ALU    : EX2;
 7088 %}
 7089 
 7090 //------- Conditional instructions ------------------------
 7091 
 7092 // Conditional no operands
 7093 // Eg.  CSINC   x0, zr, zr, <cond>
 7094 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 7095 %{
 7096   single_instruction;
 7097   cr     : EX1(read);
 7098   dst    : EX2(write);
 7099   INS01  : ISS;
 7100   ALU    : EX2;
 7101 %}
 7102 
 7103 // Conditional 2 operand
 7104 // EG.  CSEL    X0, X1, X2, <cond>
 7105 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 7106 %{
 7107   single_instruction;
 7108   cr     : EX1(read);
 7109   src1   : EX1(read);
 7110   src2   : EX1(read);
 7111   dst    : EX2(write);
 7112   INS01  : ISS;
 7113   ALU    : EX2;
 7114 %}
 7115 
 7116 // Conditional 2 operand
 7117 // EG.  CSEL    X0, X1, X2, <cond>
 7118 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 7119 %{
 7120   single_instruction;
 7121   cr     : EX1(read);
 7122   src    : EX1(read);
 7123   dst    : EX2(write);
 7124   INS01  : ISS;
 7125   ALU    : EX2;
 7126 %}
 7127 
 7128 //------- Multiply pipeline operations --------------------
 7129 
 7130 // Multiply reg-reg
 7131 // Eg.  MUL     w0, w1, w2
 7132 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7133 %{
 7134   single_instruction;
 7135   dst    : WR(write);
 7136   src1   : ISS(read);
 7137   src2   : ISS(read);
 7138   INS01  : ISS;
 7139   MAC    : WR;
 7140 %}
 7141 
 7142 // Multiply accumulate
 7143 // Eg.  MADD    w0, w1, w2, w3
 7144 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7145 %{
 7146   single_instruction;
 7147   dst    : WR(write);
 7148   src1   : ISS(read);
 7149   src2   : ISS(read);
 7150   src3   : ISS(read);
 7151   INS01  : ISS;
 7152   MAC    : WR;
 7153 %}
 7154 
 7155 // Eg.  MUL     w0, w1, w2
 7156 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7157 %{
 7158   single_instruction;
 7159   fixed_latency(3); // Maximum latency for 64 bit mul
 7160   dst    : WR(write);
 7161   src1   : ISS(read);
 7162   src2   : ISS(read);
 7163   INS01  : ISS;
 7164   MAC    : WR;
 7165 %}
 7166 
 7167 // Multiply accumulate
 7168 // Eg.  MADD    w0, w1, w2, w3
 7169 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7170 %{
 7171   single_instruction;
 7172   fixed_latency(3); // Maximum latency for 64 bit mul
 7173   dst    : WR(write);
 7174   src1   : ISS(read);
 7175   src2   : ISS(read);
 7176   src3   : ISS(read);
 7177   INS01  : ISS;
 7178   MAC    : WR;
 7179 %}
 7180 
 7181 //------- Divide pipeline operations --------------------
 7182 
 7183 // Eg.  SDIV    w0, w1, w2
 7184 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7185 %{
 7186   single_instruction;
 7187   fixed_latency(8); // Maximum latency for 32 bit divide
 7188   dst    : WR(write);
 7189   src1   : ISS(read);
 7190   src2   : ISS(read);
 7191   INS0   : ISS; // Can only dual issue as instruction 0
 7192   DIV    : WR;
 7193 %}
 7194 
 7195 // Eg.  SDIV    x0, x1, x2
 7196 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7197 %{
 7198   single_instruction;
 7199   fixed_latency(16); // Maximum latency for 64 bit divide
 7200   dst    : WR(write);
 7201   src1   : ISS(read);
 7202   src2   : ISS(read);
 7203   INS0   : ISS; // Can only dual issue as instruction 0
 7204   DIV    : WR;
 7205 %}
 7206 
 7207 //------- Load pipeline operations ------------------------
 7208 
 7209 // Load - prefetch
 7210 // Eg.  PFRM    <mem>
 7211 pipe_class iload_prefetch(memory mem)
 7212 %{
 7213   single_instruction;
 7214   mem    : ISS(read);
 7215   INS01  : ISS;
 7216   LDST   : WR;
 7217 %}
 7218 
 7219 // Load - reg, mem
 7220 // Eg.  LDR     x0, <mem>
 7221 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7222 %{
 7223   single_instruction;
 7224   dst    : WR(write);
 7225   mem    : ISS(read);
 7226   INS01  : ISS;
 7227   LDST   : WR;
 7228 %}
 7229 
 7230 // Load - reg, reg
 7231 // Eg.  LDR     x0, [sp, x1]
 7232 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7233 %{
 7234   single_instruction;
 7235   dst    : WR(write);
 7236   src    : ISS(read);
 7237   INS01  : ISS;
 7238   LDST   : WR;
 7239 %}
 7240 
 7241 //------- Store pipeline operations -----------------------
 7242 
 7243 // Store - zr, mem
 7244 // Eg.  STR     zr, <mem>
 7245 pipe_class istore_mem(memory mem)
 7246 %{
 7247   single_instruction;
 7248   mem    : ISS(read);
 7249   INS01  : ISS;
 7250   LDST   : WR;
 7251 %}
 7252 
 7253 // Store - reg, mem
 7254 // Eg.  STR     x0, <mem>
 7255 pipe_class istore_reg_mem(iRegI src, memory mem)
 7256 %{
 7257   single_instruction;
 7258   mem    : ISS(read);
 7259   src    : EX2(read);
 7260   INS01  : ISS;
 7261   LDST   : WR;
 7262 %}
 7263 
 7264 // Store - reg, reg
 7265 // Eg. STR      x0, [sp, x1]
 7266 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7267 %{
 7268   single_instruction;
 7269   dst    : ISS(read);
 7270   src    : EX2(read);
 7271   INS01  : ISS;
 7272   LDST   : WR;
 7273 %}
 7274 
 7275 //------- Store pipeline operations -----------------------
 7276 
 7277 // Branch
 7278 pipe_class pipe_branch()
 7279 %{
 7280   single_instruction;
 7281   INS01  : ISS;
 7282   BRANCH : EX1;
 7283 %}
 7284 
 7285 // Conditional branch
 7286 pipe_class pipe_branch_cond(rFlagsReg cr)
 7287 %{
 7288   single_instruction;
 7289   cr     : EX1(read);
 7290   INS01  : ISS;
 7291   BRANCH : EX1;
 7292 %}
 7293 
 7294 // Compare & Branch
 7295 // EG.  CBZ/CBNZ
 7296 pipe_class pipe_cmp_branch(iRegI op1)
 7297 %{
 7298   single_instruction;
 7299   op1    : EX1(read);
 7300   INS01  : ISS;
 7301   BRANCH : EX1;
 7302 %}
 7303 
 7304 //------- Synchronisation operations ----------------------
 7305 
 7306 // Any operation requiring serialization.
 7307 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7308 pipe_class pipe_serial()
 7309 %{
 7310   single_instruction;
 7311   force_serialization;
 7312   fixed_latency(16);
 7313   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7314   LDST   : WR;
 7315 %}
 7316 
 7317 // Generic big/slow expanded idiom - also serialized
 7318 pipe_class pipe_slow()
 7319 %{
 7320   instruction_count(10);
 7321   multiple_bundles;
 7322   force_serialization;
 7323   fixed_latency(16);
 7324   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7325   LDST   : WR;
 7326 %}
 7327 
 7328 // Empty pipeline class
 7329 pipe_class pipe_class_empty()
 7330 %{
 7331   single_instruction;
 7332   fixed_latency(0);
 7333 %}
 7334 
 7335 // Default pipeline class.
 7336 pipe_class pipe_class_default()
 7337 %{
 7338   single_instruction;
 7339   fixed_latency(2);
 7340 %}
 7341 
 7342 // Pipeline class for compares.
 7343 pipe_class pipe_class_compare()
 7344 %{
 7345   single_instruction;
 7346   fixed_latency(16);
 7347 %}
 7348 
 7349 // Pipeline class for memory operations.
 7350 pipe_class pipe_class_memory()
 7351 %{
 7352   single_instruction;
 7353   fixed_latency(16);
 7354 %}
 7355 
 7356 // Pipeline class for call.
 7357 pipe_class pipe_class_call()
 7358 %{
 7359   single_instruction;
 7360   fixed_latency(100);
 7361 %}
 7362 
 7363 // Define the class for the Nop node.
 7364 define %{
 7365    MachNop = pipe_class_empty;
 7366 %}
 7367 
 7368 %}
 7369 //----------INSTRUCTIONS-------------------------------------------------------
 7370 //
 7371 // match      -- States which machine-independent subtree may be replaced
 7372 //               by this instruction.
 7373 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7374 //               selection to identify a minimum cost tree of machine
 7375 //               instructions that matches a tree of machine-independent
 7376 //               instructions.
 7377 // format     -- A string providing the disassembly for this instruction.
 7378 //               The value of an instruction's operand may be inserted
 7379 //               by referring to it with a '$' prefix.
 7380 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7381 //               to within an encode class as $primary, $secondary, and $tertiary
 7382 //               rrspectively.  The primary opcode is commonly used to
 7383 //               indicate the type of machine instruction, while secondary
 7384 //               and tertiary are often used for prefix options or addressing
 7385 //               modes.
 7386 // ins_encode -- A list of encode classes with parameters. The encode class
 7387 //               name must have been defined in an 'enc_class' specification
 7388 //               in the encode section of the architecture description.
 7389 
 7390 // ============================================================================
 7391 // Memory (Load/Store) Instructions
 7392 
 7393 // Load Instructions
 7394 
 7395 // Load Byte (8 bit signed)
 7396 instruct loadB(iRegINoSp dst, memory1 mem)
 7397 %{
 7398   match(Set dst (LoadB mem));
 7399   predicate(!needs_acquiring_load(n));
 7400 
 7401   ins_cost(4 * INSN_COST);
 7402   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7403 
 7404   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7405 
 7406   ins_pipe(iload_reg_mem);
 7407 %}
 7408 
 7409 // Load Byte (8 bit signed) into long
 7410 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7411 %{
 7412   match(Set dst (ConvI2L (LoadB mem)));
 7413   predicate(!needs_acquiring_load(n->in(1)));
 7414 
 7415   ins_cost(4 * INSN_COST);
 7416   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7417 
 7418   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7419 
 7420   ins_pipe(iload_reg_mem);
 7421 %}
 7422 
 7423 // Load Byte (8 bit unsigned)
 7424 instruct loadUB(iRegINoSp dst, memory1 mem)
 7425 %{
 7426   match(Set dst (LoadUB mem));
 7427   predicate(!needs_acquiring_load(n));
 7428 
 7429   ins_cost(4 * INSN_COST);
 7430   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7431 
 7432   ins_encode(aarch64_enc_ldrb(dst, mem));
 7433 
 7434   ins_pipe(iload_reg_mem);
 7435 %}
 7436 
 7437 // Load Byte (8 bit unsigned) into long
 7438 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7439 %{
 7440   match(Set dst (ConvI2L (LoadUB mem)));
 7441   predicate(!needs_acquiring_load(n->in(1)));
 7442 
 7443   ins_cost(4 * INSN_COST);
 7444   format %{ "ldrb  $dst, $mem\t# byte" %}
 7445 
 7446   ins_encode(aarch64_enc_ldrb(dst, mem));
 7447 
 7448   ins_pipe(iload_reg_mem);
 7449 %}
 7450 
 7451 // Load Short (16 bit signed)
 7452 instruct loadS(iRegINoSp dst, memory2 mem)
 7453 %{
 7454   match(Set dst (LoadS mem));
 7455   predicate(!needs_acquiring_load(n));
 7456 
 7457   ins_cost(4 * INSN_COST);
 7458   format %{ "ldrshw  $dst, $mem\t# short" %}
 7459 
 7460   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7461 
 7462   ins_pipe(iload_reg_mem);
 7463 %}
 7464 
 7465 // Load Short (16 bit signed) into long
 7466 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7467 %{
 7468   match(Set dst (ConvI2L (LoadS mem)));
 7469   predicate(!needs_acquiring_load(n->in(1)));
 7470 
 7471   ins_cost(4 * INSN_COST);
 7472   format %{ "ldrsh  $dst, $mem\t# short" %}
 7473 
 7474   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7475 
 7476   ins_pipe(iload_reg_mem);
 7477 %}
 7478 
 7479 // Load Char (16 bit unsigned)
 7480 instruct loadUS(iRegINoSp dst, memory2 mem)
 7481 %{
 7482   match(Set dst (LoadUS mem));
 7483   predicate(!needs_acquiring_load(n));
 7484 
 7485   ins_cost(4 * INSN_COST);
 7486   format %{ "ldrh  $dst, $mem\t# short" %}
 7487 
 7488   ins_encode(aarch64_enc_ldrh(dst, mem));
 7489 
 7490   ins_pipe(iload_reg_mem);
 7491 %}
 7492 
 7493 // Load Short/Char (16 bit unsigned) into long
 7494 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7495 %{
 7496   match(Set dst (ConvI2L (LoadUS mem)));
 7497   predicate(!needs_acquiring_load(n->in(1)));
 7498 
 7499   ins_cost(4 * INSN_COST);
 7500   format %{ "ldrh  $dst, $mem\t# short" %}
 7501 
 7502   ins_encode(aarch64_enc_ldrh(dst, mem));
 7503 
 7504   ins_pipe(iload_reg_mem);
 7505 %}
 7506 
 7507 // Load Integer (32 bit signed)
 7508 instruct loadI(iRegINoSp dst, memory4 mem)
 7509 %{
 7510   match(Set dst (LoadI mem));
 7511   predicate(!needs_acquiring_load(n));
 7512 
 7513   ins_cost(4 * INSN_COST);
 7514   format %{ "ldrw  $dst, $mem\t# int" %}
 7515 
 7516   ins_encode(aarch64_enc_ldrw(dst, mem));
 7517 
 7518   ins_pipe(iload_reg_mem);
 7519 %}
 7520 
 7521 // Load Integer (32 bit signed) into long
 7522 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7523 %{
 7524   match(Set dst (ConvI2L (LoadI mem)));
 7525   predicate(!needs_acquiring_load(n->in(1)));
 7526 
 7527   ins_cost(4 * INSN_COST);
 7528   format %{ "ldrsw  $dst, $mem\t# int" %}
 7529 
 7530   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7531 
 7532   ins_pipe(iload_reg_mem);
 7533 %}
 7534 
 7535 // Load Integer (32 bit unsigned) into long
 7536 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7537 %{
 7538   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7539   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7540 
 7541   ins_cost(4 * INSN_COST);
 7542   format %{ "ldrw  $dst, $mem\t# int" %}
 7543 
 7544   ins_encode(aarch64_enc_ldrw(dst, mem));
 7545 
 7546   ins_pipe(iload_reg_mem);
 7547 %}
 7548 
 7549 // Load Long (64 bit signed)
 7550 instruct loadL(iRegLNoSp dst, memory8 mem)
 7551 %{
 7552   match(Set dst (LoadL mem));
 7553   predicate(!needs_acquiring_load(n));
 7554 
 7555   ins_cost(4 * INSN_COST);
 7556   format %{ "ldr  $dst, $mem\t# int" %}
 7557 
 7558   ins_encode(aarch64_enc_ldr(dst, mem));
 7559 
 7560   ins_pipe(iload_reg_mem);
 7561 %}
 7562 
 7563 // Load Range
 7564 instruct loadRange(iRegINoSp dst, memory4 mem)
 7565 %{
 7566   match(Set dst (LoadRange mem));
 7567 
 7568   ins_cost(4 * INSN_COST);
 7569   format %{ "ldrw  $dst, $mem\t# range" %}
 7570 
 7571   ins_encode(aarch64_enc_ldrw(dst, mem));
 7572 
 7573   ins_pipe(iload_reg_mem);
 7574 %}
 7575 
 7576 // Load Pointer
 7577 instruct loadP(iRegPNoSp dst, memory8 mem)
 7578 %{
 7579   match(Set dst (LoadP mem));
 7580   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7581 
 7582   ins_cost(4 * INSN_COST);
 7583   format %{ "ldr  $dst, $mem\t# ptr" %}
 7584 
 7585   ins_encode(aarch64_enc_ldr(dst, mem));
 7586 
 7587   ins_pipe(iload_reg_mem);
 7588 %}
 7589 
 7590 // Load Compressed Pointer
 7591 instruct loadN(iRegNNoSp dst, memory4 mem)
 7592 %{
 7593   match(Set dst (LoadN mem));
 7594   predicate(!needs_acquiring_load(n));
 7595 
 7596   ins_cost(4 * INSN_COST);
 7597   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7598 
 7599   ins_encode(aarch64_enc_ldrw(dst, mem));
 7600 
 7601   ins_pipe(iload_reg_mem);
 7602 %}
 7603 
 7604 // Load Klass Pointer
 7605 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7606 %{
 7607   match(Set dst (LoadKlass mem));
 7608   predicate(!needs_acquiring_load(n));
 7609 
 7610   ins_cost(4 * INSN_COST);
 7611   format %{ "ldr  $dst, $mem\t# class" %}
 7612 
 7613   ins_encode(aarch64_enc_ldr(dst, mem));
 7614 
 7615   ins_pipe(iload_reg_mem);
 7616 %}
 7617 
 7618 // Load Narrow Klass Pointer
 7619 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7620 %{
 7621   match(Set dst (LoadNKlass mem));
 7622   predicate(!needs_acquiring_load(n));
 7623 
 7624   ins_cost(4 * INSN_COST);
 7625   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7626 
 7627   ins_encode(aarch64_enc_ldrw(dst, mem));
 7628 
 7629   ins_pipe(iload_reg_mem);
 7630 %}
 7631 
 7632 // Load Float
 7633 instruct loadF(vRegF dst, memory4 mem)
 7634 %{
 7635   match(Set dst (LoadF mem));
 7636   predicate(!needs_acquiring_load(n));
 7637 
 7638   ins_cost(4 * INSN_COST);
 7639   format %{ "ldrs  $dst, $mem\t# float" %}
 7640 
 7641   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7642 
 7643   ins_pipe(pipe_class_memory);
 7644 %}
 7645 
 7646 // Load Double
 7647 instruct loadD(vRegD dst, memory8 mem)
 7648 %{
 7649   match(Set dst (LoadD mem));
 7650   predicate(!needs_acquiring_load(n));
 7651 
 7652   ins_cost(4 * INSN_COST);
 7653   format %{ "ldrd  $dst, $mem\t# double" %}
 7654 
 7655   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7656 
 7657   ins_pipe(pipe_class_memory);
 7658 %}
 7659 
 7660 
 7661 // Load Int Constant
 7662 instruct loadConI(iRegINoSp dst, immI src)
 7663 %{
 7664   match(Set dst src);
 7665 
 7666   ins_cost(INSN_COST);
 7667   format %{ "mov $dst, $src\t# int" %}
 7668 
 7669   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7670 
 7671   ins_pipe(ialu_imm);
 7672 %}
 7673 
 7674 // Load Long Constant
 7675 instruct loadConL(iRegLNoSp dst, immL src)
 7676 %{
 7677   match(Set dst src);
 7678 
 7679   ins_cost(INSN_COST);
 7680   format %{ "mov $dst, $src\t# long" %}
 7681 
 7682   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7683 
 7684   ins_pipe(ialu_imm);
 7685 %}
 7686 
 7687 // Load Pointer Constant
 7688 
 7689 instruct loadConP(iRegPNoSp dst, immP con)
 7690 %{
 7691   match(Set dst con);
 7692 
 7693   ins_cost(INSN_COST * 4);
 7694   format %{
 7695     "mov  $dst, $con\t# ptr\n\t"
 7696   %}
 7697 
 7698   ins_encode(aarch64_enc_mov_p(dst, con));
 7699 
 7700   ins_pipe(ialu_imm);
 7701 %}
 7702 
 7703 // Load Null Pointer Constant
 7704 
 7705 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7706 %{
 7707   match(Set dst con);
 7708 
 7709   ins_cost(INSN_COST);
 7710   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7711 
 7712   ins_encode(aarch64_enc_mov_p0(dst, con));
 7713 
 7714   ins_pipe(ialu_imm);
 7715 %}
 7716 
 7717 // Load Pointer Constant One
 7718 
 7719 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7720 %{
 7721   match(Set dst con);
 7722 
 7723   ins_cost(INSN_COST);
 7724   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7725 
 7726   ins_encode(aarch64_enc_mov_p1(dst, con));
 7727 
 7728   ins_pipe(ialu_imm);
 7729 %}
 7730 
 7731 // Load Byte Map Base Constant
 7732 
 7733 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7734 %{
 7735   match(Set dst con);
 7736 
 7737   ins_cost(INSN_COST);
 7738   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7739 
 7740   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7741 
 7742   ins_pipe(ialu_imm);
 7743 %}
 7744 
 7745 // Load Narrow Pointer Constant
 7746 
 7747 instruct loadConN(iRegNNoSp dst, immN con)
 7748 %{
 7749   match(Set dst con);
 7750 
 7751   ins_cost(INSN_COST * 4);
 7752   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7753 
 7754   ins_encode(aarch64_enc_mov_n(dst, con));
 7755 
 7756   ins_pipe(ialu_imm);
 7757 %}
 7758 
 7759 // Load Narrow Null Pointer Constant
 7760 
 7761 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7762 %{
 7763   match(Set dst con);
 7764 
 7765   ins_cost(INSN_COST);
 7766   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7767 
 7768   ins_encode(aarch64_enc_mov_n0(dst, con));
 7769 
 7770   ins_pipe(ialu_imm);
 7771 %}
 7772 
 7773 // Load Narrow Klass Constant
 7774 
 7775 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7776 %{
 7777   match(Set dst con);
 7778 
 7779   ins_cost(INSN_COST);
 7780   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7781 
 7782   ins_encode(aarch64_enc_mov_nk(dst, con));
 7783 
 7784   ins_pipe(ialu_imm);
 7785 %}
 7786 
 7787 // Load Packed Float Constant
 7788 
 7789 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7790   match(Set dst con);
 7791   ins_cost(INSN_COST * 4);
 7792   format %{ "fmovs  $dst, $con"%}
 7793   ins_encode %{
 7794     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7795   %}
 7796 
 7797   ins_pipe(fp_imm_s);
 7798 %}
 7799 
 7800 // Load Float Constant
 7801 
 7802 instruct loadConF(vRegF dst, immF con) %{
 7803   match(Set dst con);
 7804 
 7805   ins_cost(INSN_COST * 4);
 7806 
 7807   format %{
 7808     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7809   %}
 7810 
 7811   ins_encode %{
 7812     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7813   %}
 7814 
 7815   ins_pipe(fp_load_constant_s);
 7816 %}
 7817 
 7818 // Load Packed Double Constant
 7819 
 7820 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7821   match(Set dst con);
 7822   ins_cost(INSN_COST);
 7823   format %{ "fmovd  $dst, $con"%}
 7824   ins_encode %{
 7825     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7826   %}
 7827 
 7828   ins_pipe(fp_imm_d);
 7829 %}
 7830 
 7831 // Load Double Constant
 7832 
 7833 instruct loadConD(vRegD dst, immD con) %{
 7834   match(Set dst con);
 7835 
 7836   ins_cost(INSN_COST * 5);
 7837   format %{
 7838     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7839   %}
 7840 
 7841   ins_encode %{
 7842     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7843   %}
 7844 
 7845   ins_pipe(fp_load_constant_d);
 7846 %}
 7847 
 7848 // Store Instructions
 7849 
 7850 // Store CMS card-mark Immediate
 7851 instruct storeimmCM0(immI0 zero, memory1 mem)
 7852 %{
 7853   match(Set mem (StoreCM mem zero));
 7854 
 7855   ins_cost(INSN_COST);
 7856   format %{ "storestore (elided)\n\t"
 7857             "strb zr, $mem\t# byte" %}
 7858 
 7859   ins_encode(aarch64_enc_strb0(mem));
 7860 
 7861   ins_pipe(istore_mem);
 7862 %}
 7863 
 7864 // Store CMS card-mark Immediate with intervening StoreStore
 7865 // needed when using CMS with no conditional card marking
 7866 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7867 %{
 7868   match(Set mem (StoreCM mem zero));
 7869 
 7870   ins_cost(INSN_COST * 2);
 7871   format %{ "storestore\n\t"
 7872             "dmb ishst"
 7873             "\n\tstrb zr, $mem\t# byte" %}
 7874 
 7875   ins_encode(aarch64_enc_strb0_ordered(mem));
 7876 
 7877   ins_pipe(istore_mem);
 7878 %}
 7879 
 7880 // Store Byte
 7881 instruct storeB(iRegIorL2I src, memory1 mem)
 7882 %{
 7883   match(Set mem (StoreB mem src));
 7884   predicate(!needs_releasing_store(n));
 7885 
 7886   ins_cost(INSN_COST);
 7887   format %{ "strb  $src, $mem\t# byte" %}
 7888 
 7889   ins_encode(aarch64_enc_strb(src, mem));
 7890 
 7891   ins_pipe(istore_reg_mem);
 7892 %}
 7893 
 7894 
 7895 instruct storeimmB0(immI0 zero, memory1 mem)
 7896 %{
 7897   match(Set mem (StoreB mem zero));
 7898   predicate(!needs_releasing_store(n));
 7899 
 7900   ins_cost(INSN_COST);
 7901   format %{ "strb rscractch2, $mem\t# byte" %}
 7902 
 7903   ins_encode(aarch64_enc_strb0(mem));
 7904 
 7905   ins_pipe(istore_mem);
 7906 %}
 7907 
 7908 // Store Char/Short
 7909 instruct storeC(iRegIorL2I src, memory2 mem)
 7910 %{
 7911   match(Set mem (StoreC mem src));
 7912   predicate(!needs_releasing_store(n));
 7913 
 7914   ins_cost(INSN_COST);
 7915   format %{ "strh  $src, $mem\t# short" %}
 7916 
 7917   ins_encode(aarch64_enc_strh(src, mem));
 7918 
 7919   ins_pipe(istore_reg_mem);
 7920 %}
 7921 
 7922 instruct storeimmC0(immI0 zero, memory2 mem)
 7923 %{
 7924   match(Set mem (StoreC mem zero));
 7925   predicate(!needs_releasing_store(n));
 7926 
 7927   ins_cost(INSN_COST);
 7928   format %{ "strh  zr, $mem\t# short" %}
 7929 
 7930   ins_encode(aarch64_enc_strh0(mem));
 7931 
 7932   ins_pipe(istore_mem);
 7933 %}
 7934 
 7935 // Store Integer
 7936 
 7937 instruct storeI(iRegIorL2I src, memory4 mem)
 7938 %{
 7939   match(Set mem(StoreI mem src));
 7940   predicate(!needs_releasing_store(n));
 7941 
 7942   ins_cost(INSN_COST);
 7943   format %{ "strw  $src, $mem\t# int" %}
 7944 
 7945   ins_encode(aarch64_enc_strw(src, mem));
 7946 
 7947   ins_pipe(istore_reg_mem);
 7948 %}
 7949 
 7950 instruct storeimmI0(immI0 zero, memory4 mem)
 7951 %{
 7952   match(Set mem(StoreI mem zero));
 7953   predicate(!needs_releasing_store(n));
 7954 
 7955   ins_cost(INSN_COST);
 7956   format %{ "strw  zr, $mem\t# int" %}
 7957 
 7958   ins_encode(aarch64_enc_strw0(mem));
 7959 
 7960   ins_pipe(istore_mem);
 7961 %}
 7962 
 7963 // Store Long (64 bit signed)
 7964 instruct storeL(iRegL src, memory8 mem)
 7965 %{
 7966   match(Set mem (StoreL mem src));
 7967   predicate(!needs_releasing_store(n));
 7968 
 7969   ins_cost(INSN_COST);
 7970   format %{ "str  $src, $mem\t# int" %}
 7971 
 7972   ins_encode(aarch64_enc_str(src, mem));
 7973 
 7974   ins_pipe(istore_reg_mem);
 7975 %}
 7976 
 7977 // Store Long (64 bit signed)
 7978 instruct storeimmL0(immL0 zero, memory8 mem)
 7979 %{
 7980   match(Set mem (StoreL mem zero));
 7981   predicate(!needs_releasing_store(n));
 7982 
 7983   ins_cost(INSN_COST);
 7984   format %{ "str  zr, $mem\t# int" %}
 7985 
 7986   ins_encode(aarch64_enc_str0(mem));
 7987 
 7988   ins_pipe(istore_mem);
 7989 %}
 7990 
 7991 // Store Pointer
 7992 instruct storeP(iRegP src, memory8 mem)
 7993 %{
 7994   match(Set mem (StoreP mem src));
 7995   predicate(!needs_releasing_store(n));
 7996 
 7997   ins_cost(INSN_COST);
 7998   format %{ "str  $src, $mem\t# ptr" %}
 7999 
 8000   ins_encode(aarch64_enc_str(src, mem));
 8001 
 8002   ins_pipe(istore_reg_mem);
 8003 %}
 8004 
 8005 // Store Pointer
 8006 instruct storeimmP0(immP0 zero, memory8 mem)
 8007 %{
 8008   match(Set mem (StoreP mem zero));
 8009   predicate(!needs_releasing_store(n));
 8010 
 8011   ins_cost(INSN_COST);
 8012   format %{ "str zr, $mem\t# ptr" %}
 8013 
 8014   ins_encode(aarch64_enc_str0(mem));
 8015 
 8016   ins_pipe(istore_mem);
 8017 %}
 8018 
 8019 // Store Compressed Pointer
 8020 instruct storeN(iRegN src, memory4 mem)
 8021 %{
 8022   match(Set mem (StoreN mem src));
 8023   predicate(!needs_releasing_store(n));
 8024 
 8025   ins_cost(INSN_COST);
 8026   format %{ "strw  $src, $mem\t# compressed ptr" %}
 8027 
 8028   ins_encode(aarch64_enc_strw(src, mem));
 8029 
 8030   ins_pipe(istore_reg_mem);
 8031 %}
 8032 
 8033 instruct storeImmN0(immN0 zero, memory4 mem)
 8034 %{
 8035   match(Set mem (StoreN mem zero));
 8036   predicate(!needs_releasing_store(n));
 8037 
 8038   ins_cost(INSN_COST);
 8039   format %{ "strw  zr, $mem\t# compressed ptr" %}
 8040 
 8041   ins_encode(aarch64_enc_strw0(mem));
 8042 
 8043   ins_pipe(istore_mem);
 8044 %}
 8045 
 8046 // Store Float
 8047 instruct storeF(vRegF src, memory4 mem)
 8048 %{
 8049   match(Set mem (StoreF mem src));
 8050   predicate(!needs_releasing_store(n));
 8051 
 8052   ins_cost(INSN_COST);
 8053   format %{ "strs  $src, $mem\t# float" %}
 8054 
 8055   ins_encode( aarch64_enc_strs(src, mem) );
 8056 
 8057   ins_pipe(pipe_class_memory);
 8058 %}
 8059 
 8060 // TODO
 8061 // implement storeImmF0 and storeFImmPacked
 8062 
 8063 // Store Double
 8064 instruct storeD(vRegD src, memory8 mem)
 8065 %{
 8066   match(Set mem (StoreD mem src));
 8067   predicate(!needs_releasing_store(n));
 8068 
 8069   ins_cost(INSN_COST);
 8070   format %{ "strd  $src, $mem\t# double" %}
 8071 
 8072   ins_encode( aarch64_enc_strd(src, mem) );
 8073 
 8074   ins_pipe(pipe_class_memory);
 8075 %}
 8076 
 8077 // Store Compressed Klass Pointer
 8078 instruct storeNKlass(iRegN src, memory4 mem)
 8079 %{
 8080   predicate(!needs_releasing_store(n));
 8081   match(Set mem (StoreNKlass mem src));
 8082 
 8083   ins_cost(INSN_COST);
 8084   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 8085 
 8086   ins_encode(aarch64_enc_strw(src, mem));
 8087 
 8088   ins_pipe(istore_reg_mem);
 8089 %}
 8090 
 8091 // TODO
 8092 // implement storeImmD0 and storeDImmPacked
 8093 
 8094 // prefetch instructions
 8095 // Must be safe to execute with invalid address (cannot fault).
 8096 
 8097 instruct prefetchalloc( memory8 mem ) %{
 8098   match(PrefetchAllocation mem);
 8099 
 8100   ins_cost(INSN_COST);
 8101   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 8102 
 8103   ins_encode( aarch64_enc_prefetchw(mem) );
 8104 
 8105   ins_pipe(iload_prefetch);
 8106 %}
 8107 
 8108 //  ---------------- volatile loads and stores ----------------
 8109 
 8110 // Load Byte (8 bit signed)
 8111 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8112 %{
 8113   match(Set dst (LoadB mem));
 8114 
 8115   ins_cost(VOLATILE_REF_COST);
 8116   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8117 
 8118   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8119 
 8120   ins_pipe(pipe_serial);
 8121 %}
 8122 
 8123 // Load Byte (8 bit signed) into long
 8124 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8125 %{
 8126   match(Set dst (ConvI2L (LoadB mem)));
 8127 
 8128   ins_cost(VOLATILE_REF_COST);
 8129   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8130 
 8131   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8132 
 8133   ins_pipe(pipe_serial);
 8134 %}
 8135 
 8136 // Load Byte (8 bit unsigned)
 8137 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8138 %{
 8139   match(Set dst (LoadUB mem));
 8140 
 8141   ins_cost(VOLATILE_REF_COST);
 8142   format %{ "ldarb  $dst, $mem\t# byte" %}
 8143 
 8144   ins_encode(aarch64_enc_ldarb(dst, mem));
 8145 
 8146   ins_pipe(pipe_serial);
 8147 %}
 8148 
 8149 // Load Byte (8 bit unsigned) into long
 8150 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8151 %{
 8152   match(Set dst (ConvI2L (LoadUB mem)));
 8153 
 8154   ins_cost(VOLATILE_REF_COST);
 8155   format %{ "ldarb  $dst, $mem\t# byte" %}
 8156 
 8157   ins_encode(aarch64_enc_ldarb(dst, mem));
 8158 
 8159   ins_pipe(pipe_serial);
 8160 %}
 8161 
 8162 // Load Short (16 bit signed)
 8163 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8164 %{
 8165   match(Set dst (LoadS mem));
 8166 
 8167   ins_cost(VOLATILE_REF_COST);
 8168   format %{ "ldarshw  $dst, $mem\t# short" %}
 8169 
 8170   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8171 
 8172   ins_pipe(pipe_serial);
 8173 %}
 8174 
 8175 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8176 %{
 8177   match(Set dst (LoadUS mem));
 8178 
 8179   ins_cost(VOLATILE_REF_COST);
 8180   format %{ "ldarhw  $dst, $mem\t# short" %}
 8181 
 8182   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8183 
 8184   ins_pipe(pipe_serial);
 8185 %}
 8186 
 8187 // Load Short/Char (16 bit unsigned) into long
 8188 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8189 %{
 8190   match(Set dst (ConvI2L (LoadUS mem)));
 8191 
 8192   ins_cost(VOLATILE_REF_COST);
 8193   format %{ "ldarh  $dst, $mem\t# short" %}
 8194 
 8195   ins_encode(aarch64_enc_ldarh(dst, mem));
 8196 
 8197   ins_pipe(pipe_serial);
 8198 %}
 8199 
 8200 // Load Short/Char (16 bit signed) into long
 8201 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8202 %{
 8203   match(Set dst (ConvI2L (LoadS mem)));
 8204 
 8205   ins_cost(VOLATILE_REF_COST);
 8206   format %{ "ldarh  $dst, $mem\t# short" %}
 8207 
 8208   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8209 
 8210   ins_pipe(pipe_serial);
 8211 %}
 8212 
 8213 // Load Integer (32 bit signed)
 8214 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8215 %{
 8216   match(Set dst (LoadI mem));
 8217 
 8218   ins_cost(VOLATILE_REF_COST);
 8219   format %{ "ldarw  $dst, $mem\t# int" %}
 8220 
 8221   ins_encode(aarch64_enc_ldarw(dst, mem));
 8222 
 8223   ins_pipe(pipe_serial);
 8224 %}
 8225 
 8226 // Load Integer (32 bit unsigned) into long
 8227 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8228 %{
 8229   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8230 
 8231   ins_cost(VOLATILE_REF_COST);
 8232   format %{ "ldarw  $dst, $mem\t# int" %}
 8233 
 8234   ins_encode(aarch64_enc_ldarw(dst, mem));
 8235 
 8236   ins_pipe(pipe_serial);
 8237 %}
 8238 
 8239 // Load Long (64 bit signed)
 8240 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8241 %{
 8242   match(Set dst (LoadL mem));
 8243 
 8244   ins_cost(VOLATILE_REF_COST);
 8245   format %{ "ldar  $dst, $mem\t# int" %}
 8246 
 8247   ins_encode(aarch64_enc_ldar(dst, mem));
 8248 
 8249   ins_pipe(pipe_serial);
 8250 %}
 8251 
 8252 // Load Pointer
 8253 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8254 %{
 8255   match(Set dst (LoadP mem));
 8256   predicate(n->as_Load()->barrier_data() == 0);
 8257 
 8258   ins_cost(VOLATILE_REF_COST);
 8259   format %{ "ldar  $dst, $mem\t# ptr" %}
 8260 
 8261   ins_encode(aarch64_enc_ldar(dst, mem));
 8262 
 8263   ins_pipe(pipe_serial);
 8264 %}
 8265 
 8266 // Load Compressed Pointer
 8267 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8268 %{
 8269   match(Set dst (LoadN mem));
 8270 
 8271   ins_cost(VOLATILE_REF_COST);
 8272   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8273 
 8274   ins_encode(aarch64_enc_ldarw(dst, mem));
 8275 
 8276   ins_pipe(pipe_serial);
 8277 %}
 8278 
 8279 // Load Float
 8280 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8281 %{
 8282   match(Set dst (LoadF mem));
 8283 
 8284   ins_cost(VOLATILE_REF_COST);
 8285   format %{ "ldars  $dst, $mem\t# float" %}
 8286 
 8287   ins_encode( aarch64_enc_fldars(dst, mem) );
 8288 
 8289   ins_pipe(pipe_serial);
 8290 %}
 8291 
 8292 // Load Double
 8293 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8294 %{
 8295   match(Set dst (LoadD mem));
 8296 
 8297   ins_cost(VOLATILE_REF_COST);
 8298   format %{ "ldard  $dst, $mem\t# double" %}
 8299 
 8300   ins_encode( aarch64_enc_fldard(dst, mem) );
 8301 
 8302   ins_pipe(pipe_serial);
 8303 %}
 8304 
 8305 // Store Byte
 8306 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8307 %{
 8308   match(Set mem (StoreB mem src));
 8309 
 8310   ins_cost(VOLATILE_REF_COST);
 8311   format %{ "stlrb  $src, $mem\t# byte" %}
 8312 
 8313   ins_encode(aarch64_enc_stlrb(src, mem));
 8314 
 8315   ins_pipe(pipe_class_memory);
 8316 %}
 8317 
 8318 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8319 %{
 8320   match(Set mem (StoreB mem zero));
 8321 
 8322   ins_cost(VOLATILE_REF_COST);
 8323   format %{ "stlrb  zr, $mem\t# byte" %}
 8324 
 8325   ins_encode(aarch64_enc_stlrb0(mem));
 8326 
 8327   ins_pipe(pipe_class_memory);
 8328 %}
 8329 
 8330 // Store Char/Short
 8331 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8332 %{
 8333   match(Set mem (StoreC mem src));
 8334 
 8335   ins_cost(VOLATILE_REF_COST);
 8336   format %{ "stlrh  $src, $mem\t# short" %}
 8337 
 8338   ins_encode(aarch64_enc_stlrh(src, mem));
 8339 
 8340   ins_pipe(pipe_class_memory);
 8341 %}
 8342 
 8343 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8344 %{
 8345   match(Set mem (StoreC mem zero));
 8346 
 8347   ins_cost(VOLATILE_REF_COST);
 8348   format %{ "stlrh  zr, $mem\t# short" %}
 8349 
 8350   ins_encode(aarch64_enc_stlrh0(mem));
 8351 
 8352   ins_pipe(pipe_class_memory);
 8353 %}
 8354 
 8355 // Store Integer
 8356 
 8357 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8358 %{
 8359   match(Set mem(StoreI mem src));
 8360 
 8361   ins_cost(VOLATILE_REF_COST);
 8362   format %{ "stlrw  $src, $mem\t# int" %}
 8363 
 8364   ins_encode(aarch64_enc_stlrw(src, mem));
 8365 
 8366   ins_pipe(pipe_class_memory);
 8367 %}
 8368 
 8369 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8370 %{
 8371   match(Set mem(StoreI mem zero));
 8372 
 8373   ins_cost(VOLATILE_REF_COST);
 8374   format %{ "stlrw  zr, $mem\t# int" %}
 8375 
 8376   ins_encode(aarch64_enc_stlrw0(mem));
 8377 
 8378   ins_pipe(pipe_class_memory);
 8379 %}
 8380 
 8381 // Store Long (64 bit signed)
 8382 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8383 %{
 8384   match(Set mem (StoreL mem src));
 8385 
 8386   ins_cost(VOLATILE_REF_COST);
 8387   format %{ "stlr  $src, $mem\t# int" %}
 8388 
 8389   ins_encode(aarch64_enc_stlr(src, mem));
 8390 
 8391   ins_pipe(pipe_class_memory);
 8392 %}
 8393 
 8394 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8395 %{
 8396   match(Set mem (StoreL mem zero));
 8397 
 8398   ins_cost(VOLATILE_REF_COST);
 8399   format %{ "stlr  zr, $mem\t# int" %}
 8400 
 8401   ins_encode(aarch64_enc_stlr0(mem));
 8402 
 8403   ins_pipe(pipe_class_memory);
 8404 %}
 8405 
 8406 // Store Pointer
 8407 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8408 %{
 8409   match(Set mem (StoreP mem src));
 8410 
 8411   ins_cost(VOLATILE_REF_COST);
 8412   format %{ "stlr  $src, $mem\t# ptr" %}
 8413 
 8414   ins_encode(aarch64_enc_stlr(src, mem));
 8415 
 8416   ins_pipe(pipe_class_memory);
 8417 %}
 8418 
 8419 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8420 %{
 8421   match(Set mem (StoreP mem zero));
 8422 
 8423   ins_cost(VOLATILE_REF_COST);
 8424   format %{ "stlr  zr, $mem\t# ptr" %}
 8425 
 8426   ins_encode(aarch64_enc_stlr0(mem));
 8427 
 8428   ins_pipe(pipe_class_memory);
 8429 %}
 8430 
 8431 // Store Compressed Pointer
 8432 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8433 %{
 8434   match(Set mem (StoreN mem src));
 8435 
 8436   ins_cost(VOLATILE_REF_COST);
 8437   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8438 
 8439   ins_encode(aarch64_enc_stlrw(src, mem));
 8440 
 8441   ins_pipe(pipe_class_memory);
 8442 %}
 8443 
 8444 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8445 %{
 8446   match(Set mem (StoreN mem zero));
 8447 
 8448   ins_cost(VOLATILE_REF_COST);
 8449   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8450 
 8451   ins_encode(aarch64_enc_stlrw0(mem));
 8452 
 8453   ins_pipe(pipe_class_memory);
 8454 %}
 8455 
 8456 // Store Float
 8457 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8458 %{
 8459   match(Set mem (StoreF mem src));
 8460 
 8461   ins_cost(VOLATILE_REF_COST);
 8462   format %{ "stlrs  $src, $mem\t# float" %}
 8463 
 8464   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8465 
 8466   ins_pipe(pipe_class_memory);
 8467 %}
 8468 
 8469 // TODO
 8470 // implement storeImmF0 and storeFImmPacked
 8471 
 8472 // Store Double
 8473 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8474 %{
 8475   match(Set mem (StoreD mem src));
 8476 
 8477   ins_cost(VOLATILE_REF_COST);
 8478   format %{ "stlrd  $src, $mem\t# double" %}
 8479 
 8480   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8481 
 8482   ins_pipe(pipe_class_memory);
 8483 %}
 8484 
 8485 //  ---------------- end of volatile loads and stores ----------------
 8486 
 8487 instruct cacheWB(indirect addr)
 8488 %{
 8489   predicate(VM_Version::supports_data_cache_line_flush());
 8490   match(CacheWB addr);
 8491 
 8492   ins_cost(100);
 8493   format %{"cache wb $addr" %}
 8494   ins_encode %{
 8495     assert($addr->index_position() < 0, "should be");
 8496     assert($addr$$disp == 0, "should be");
 8497     __ cache_wb(Address($addr$$base$$Register, 0));
 8498   %}
 8499   ins_pipe(pipe_slow); // XXX
 8500 %}
 8501 
 8502 instruct cacheWBPreSync()
 8503 %{
 8504   predicate(VM_Version::supports_data_cache_line_flush());
 8505   match(CacheWBPreSync);
 8506 
 8507   ins_cost(100);
 8508   format %{"cache wb presync" %}
 8509   ins_encode %{
 8510     __ cache_wbsync(true);
 8511   %}
 8512   ins_pipe(pipe_slow); // XXX
 8513 %}
 8514 
 8515 instruct cacheWBPostSync()
 8516 %{
 8517   predicate(VM_Version::supports_data_cache_line_flush());
 8518   match(CacheWBPostSync);
 8519 
 8520   ins_cost(100);
 8521   format %{"cache wb postsync" %}
 8522   ins_encode %{
 8523     __ cache_wbsync(false);
 8524   %}
 8525   ins_pipe(pipe_slow); // XXX
 8526 %}
 8527 
 8528 // ============================================================================
 8529 // BSWAP Instructions
 8530 
 8531 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8532   match(Set dst (ReverseBytesI src));
 8533 
 8534   ins_cost(INSN_COST);
 8535   format %{ "revw  $dst, $src" %}
 8536 
 8537   ins_encode %{
 8538     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8539   %}
 8540 
 8541   ins_pipe(ialu_reg);
 8542 %}
 8543 
 8544 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8545   match(Set dst (ReverseBytesL src));
 8546 
 8547   ins_cost(INSN_COST);
 8548   format %{ "rev  $dst, $src" %}
 8549 
 8550   ins_encode %{
 8551     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8552   %}
 8553 
 8554   ins_pipe(ialu_reg);
 8555 %}
 8556 
 8557 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8558   match(Set dst (ReverseBytesUS src));
 8559 
 8560   ins_cost(INSN_COST);
 8561   format %{ "rev16w  $dst, $src" %}
 8562 
 8563   ins_encode %{
 8564     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8565   %}
 8566 
 8567   ins_pipe(ialu_reg);
 8568 %}
 8569 
 8570 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8571   match(Set dst (ReverseBytesS src));
 8572 
 8573   ins_cost(INSN_COST);
 8574   format %{ "rev16w  $dst, $src\n\t"
 8575             "sbfmw $dst, $dst, #0, #15" %}
 8576 
 8577   ins_encode %{
 8578     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8579     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8580   %}
 8581 
 8582   ins_pipe(ialu_reg);
 8583 %}
 8584 
 8585 // ============================================================================
 8586 // Zero Count Instructions
 8587 
 8588 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8589   match(Set dst (CountLeadingZerosI src));
 8590 
 8591   ins_cost(INSN_COST);
 8592   format %{ "clzw  $dst, $src" %}
 8593   ins_encode %{
 8594     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8595   %}
 8596 
 8597   ins_pipe(ialu_reg);
 8598 %}
 8599 
 8600 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8601   match(Set dst (CountLeadingZerosL src));
 8602 
 8603   ins_cost(INSN_COST);
 8604   format %{ "clz   $dst, $src" %}
 8605   ins_encode %{
 8606     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8607   %}
 8608 
 8609   ins_pipe(ialu_reg);
 8610 %}
 8611 
 8612 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8613   match(Set dst (CountTrailingZerosI src));
 8614 
 8615   ins_cost(INSN_COST * 2);
 8616   format %{ "rbitw  $dst, $src\n\t"
 8617             "clzw   $dst, $dst" %}
 8618   ins_encode %{
 8619     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8620     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8621   %}
 8622 
 8623   ins_pipe(ialu_reg);
 8624 %}
 8625 
 8626 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8627   match(Set dst (CountTrailingZerosL src));
 8628 
 8629   ins_cost(INSN_COST * 2);
 8630   format %{ "rbit   $dst, $src\n\t"
 8631             "clz    $dst, $dst" %}
 8632   ins_encode %{
 8633     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8634     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8635   %}
 8636 
 8637   ins_pipe(ialu_reg);
 8638 %}
 8639 
 8640 //---------- Population Count Instructions -------------------------------------
 8641 //
 8642 
 8643 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8644   predicate(UsePopCountInstruction);
 8645   match(Set dst (PopCountI src));
 8646   effect(TEMP tmp);
 8647   ins_cost(INSN_COST * 13);
 8648 
 8649   format %{ "movw   $src, $src\n\t"
 8650             "mov    $tmp, $src\t# vector (1D)\n\t"
 8651             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8652             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8653             "mov    $dst, $tmp\t# vector (1D)" %}
 8654   ins_encode %{
 8655     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8656     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8657     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8658     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8659     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8660   %}
 8661 
 8662   ins_pipe(pipe_class_default);
 8663 %}
 8664 
 8665 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8666   predicate(UsePopCountInstruction);
 8667   match(Set dst (PopCountI (LoadI mem)));
 8668   effect(TEMP tmp);
 8669   ins_cost(INSN_COST * 13);
 8670 
 8671   format %{ "ldrs   $tmp, $mem\n\t"
 8672             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8673             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8674             "mov    $dst, $tmp\t# vector (1D)" %}
 8675   ins_encode %{
 8676     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8677     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8678               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8679     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8680     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8681     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8682   %}
 8683 
 8684   ins_pipe(pipe_class_default);
 8685 %}
 8686 
 8687 // Note: Long.bitCount(long) returns an int.
 8688 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8689   predicate(UsePopCountInstruction);
 8690   match(Set dst (PopCountL src));
 8691   effect(TEMP tmp);
 8692   ins_cost(INSN_COST * 13);
 8693 
 8694   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8695             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8696             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8697             "mov    $dst, $tmp\t# vector (1D)" %}
 8698   ins_encode %{
 8699     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8700     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8701     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8702     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8703   %}
 8704 
 8705   ins_pipe(pipe_class_default);
 8706 %}
 8707 
 8708 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8709   predicate(UsePopCountInstruction);
 8710   match(Set dst (PopCountL (LoadL mem)));
 8711   effect(TEMP tmp);
 8712   ins_cost(INSN_COST * 13);
 8713 
 8714   format %{ "ldrd   $tmp, $mem\n\t"
 8715             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8716             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8717             "mov    $dst, $tmp\t# vector (1D)" %}
 8718   ins_encode %{
 8719     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8720     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8721               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8722     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8723     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8724     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8725   %}
 8726 
 8727   ins_pipe(pipe_class_default);
 8728 %}
 8729 
 8730 // ============================================================================
 8731 // MemBar Instruction
 8732 
 8733 instruct load_fence() %{
 8734   match(LoadFence);
 8735   ins_cost(VOLATILE_REF_COST);
 8736 
 8737   format %{ "load_fence" %}
 8738 
 8739   ins_encode %{
 8740     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8741   %}
 8742   ins_pipe(pipe_serial);
 8743 %}
 8744 
 8745 instruct unnecessary_membar_acquire() %{
 8746   predicate(unnecessary_acquire(n));
 8747   match(MemBarAcquire);
 8748   ins_cost(0);
 8749 
 8750   format %{ "membar_acquire (elided)" %}
 8751 
 8752   ins_encode %{
 8753     __ block_comment("membar_acquire (elided)");
 8754   %}
 8755 
 8756   ins_pipe(pipe_class_empty);
 8757 %}
 8758 
 8759 instruct membar_acquire() %{
 8760   match(MemBarAcquire);
 8761   ins_cost(VOLATILE_REF_COST);
 8762 
 8763   format %{ "membar_acquire\n\t"
 8764             "dmb ish" %}
 8765 
 8766   ins_encode %{
 8767     __ block_comment("membar_acquire");
 8768     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8769   %}
 8770 
 8771   ins_pipe(pipe_serial);
 8772 %}
 8773 
 8774 
 8775 instruct membar_acquire_lock() %{
 8776   match(MemBarAcquireLock);
 8777   ins_cost(VOLATILE_REF_COST);
 8778 
 8779   format %{ "membar_acquire_lock (elided)" %}
 8780 
 8781   ins_encode %{
 8782     __ block_comment("membar_acquire_lock (elided)");
 8783   %}
 8784 
 8785   ins_pipe(pipe_serial);
 8786 %}
 8787 
 8788 instruct store_fence() %{
 8789   match(StoreFence);
 8790   ins_cost(VOLATILE_REF_COST);
 8791 
 8792   format %{ "store_fence" %}
 8793 
 8794   ins_encode %{
 8795     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8796   %}
 8797   ins_pipe(pipe_serial);
 8798 %}
 8799 
 8800 instruct unnecessary_membar_release() %{
 8801   predicate(unnecessary_release(n));
 8802   match(MemBarRelease);
 8803   ins_cost(0);
 8804 
 8805   format %{ "membar_release (elided)" %}
 8806 
 8807   ins_encode %{
 8808     __ block_comment("membar_release (elided)");
 8809   %}
 8810   ins_pipe(pipe_serial);
 8811 %}
 8812 
 8813 instruct membar_release() %{
 8814   match(MemBarRelease);
 8815   ins_cost(VOLATILE_REF_COST);
 8816 
 8817   format %{ "membar_release\n\t"
 8818             "dmb ish" %}
 8819 
 8820   ins_encode %{
 8821     __ block_comment("membar_release");
 8822     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8823   %}
 8824   ins_pipe(pipe_serial);
 8825 %}
 8826 
 8827 instruct membar_storestore() %{
 8828   match(MemBarStoreStore);
 8829   match(StoreStoreFence);
 8830   ins_cost(VOLATILE_REF_COST);
 8831 
 8832   format %{ "MEMBAR-store-store" %}
 8833 
 8834   ins_encode %{
 8835     __ membar(Assembler::StoreStore);
 8836   %}
 8837   ins_pipe(pipe_serial);
 8838 %}
 8839 
 8840 instruct membar_release_lock() %{
 8841   match(MemBarReleaseLock);
 8842   ins_cost(VOLATILE_REF_COST);
 8843 
 8844   format %{ "membar_release_lock (elided)" %}
 8845 
 8846   ins_encode %{
 8847     __ block_comment("membar_release_lock (elided)");
 8848   %}
 8849 
 8850   ins_pipe(pipe_serial);
 8851 %}
 8852 
 8853 instruct unnecessary_membar_volatile() %{
 8854   predicate(unnecessary_volatile(n));
 8855   match(MemBarVolatile);
 8856   ins_cost(0);
 8857 
 8858   format %{ "membar_volatile (elided)" %}
 8859 
 8860   ins_encode %{
 8861     __ block_comment("membar_volatile (elided)");
 8862   %}
 8863 
 8864   ins_pipe(pipe_serial);
 8865 %}
 8866 
 8867 instruct membar_volatile() %{
 8868   match(MemBarVolatile);
 8869   ins_cost(VOLATILE_REF_COST*100);
 8870 
 8871   format %{ "membar_volatile\n\t"
 8872              "dmb ish"%}
 8873 
 8874   ins_encode %{
 8875     __ block_comment("membar_volatile");
 8876     __ membar(Assembler::StoreLoad);
 8877   %}
 8878 
 8879   ins_pipe(pipe_serial);
 8880 %}
 8881 
 8882 // ============================================================================
 8883 // Cast/Convert Instructions
 8884 
 8885 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8886   match(Set dst (CastX2P src));
 8887 
 8888   ins_cost(INSN_COST);
 8889   format %{ "mov $dst, $src\t# long -> ptr" %}
 8890 
 8891   ins_encode %{
 8892     if ($dst$$reg != $src$$reg) {
 8893       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8894     }
 8895   %}
 8896 
 8897   ins_pipe(ialu_reg);
 8898 %}
 8899 
 8900 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8901   match(Set dst (CastP2X src));
 8902 
 8903   ins_cost(INSN_COST);
 8904   format %{ "mov $dst, $src\t# ptr -> long" %}
 8905 
 8906   ins_encode %{
 8907     if ($dst$$reg != $src$$reg) {
 8908       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8909     }
 8910   %}
 8911 
 8912   ins_pipe(ialu_reg);
 8913 %}
 8914 
 8915 // Convert oop into int for vectors alignment masking
 8916 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8917   match(Set dst (ConvL2I (CastP2X src)));
 8918 
 8919   ins_cost(INSN_COST);
 8920   format %{ "movw $dst, $src\t# ptr -> int" %}
 8921   ins_encode %{
 8922     __ movw($dst$$Register, $src$$Register);
 8923   %}
 8924 
 8925   ins_pipe(ialu_reg);
 8926 %}
 8927 
 8928 // Convert compressed oop into int for vectors alignment masking
 8929 // in case of 32bit oops (heap < 4Gb).
 8930 instruct convN2I(iRegINoSp dst, iRegN src)
 8931 %{
 8932   predicate(CompressedOops::shift() == 0);
 8933   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8934 
 8935   ins_cost(INSN_COST);
 8936   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8937   ins_encode %{
 8938     __ movw($dst$$Register, $src$$Register);
 8939   %}
 8940 
 8941   ins_pipe(ialu_reg);
 8942 %}
 8943 
 8944 
 8945 // Convert oop pointer into compressed form
 8946 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8947   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8948   match(Set dst (EncodeP src));
 8949   effect(KILL cr);
 8950   ins_cost(INSN_COST * 3);
 8951   format %{ "encode_heap_oop $dst, $src" %}
 8952   ins_encode %{
 8953     Register s = $src$$Register;
 8954     Register d = $dst$$Register;
 8955     __ encode_heap_oop(d, s);
 8956   %}
 8957   ins_pipe(ialu_reg);
 8958 %}
 8959 
 8960 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8961   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8962   match(Set dst (EncodeP src));
 8963   ins_cost(INSN_COST * 3);
 8964   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8965   ins_encode %{
 8966     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8967   %}
 8968   ins_pipe(ialu_reg);
 8969 %}
 8970 
 8971 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8972   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8973             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8974   match(Set dst (DecodeN src));
 8975   ins_cost(INSN_COST * 3);
 8976   format %{ "decode_heap_oop $dst, $src" %}
 8977   ins_encode %{
 8978     Register s = $src$$Register;
 8979     Register d = $dst$$Register;
 8980     __ decode_heap_oop(d, s);
 8981   %}
 8982   ins_pipe(ialu_reg);
 8983 %}
 8984 
 8985 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8986   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8987             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8988   match(Set dst (DecodeN src));
 8989   ins_cost(INSN_COST * 3);
 8990   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8991   ins_encode %{
 8992     Register s = $src$$Register;
 8993     Register d = $dst$$Register;
 8994     __ decode_heap_oop_not_null(d, s);
 8995   %}
 8996   ins_pipe(ialu_reg);
 8997 %}
 8998 
 8999 // n.b. AArch64 implementations of encode_klass_not_null and
 9000 // decode_klass_not_null do not modify the flags register so, unlike
 9001 // Intel, we don't kill CR as a side effect here
 9002 
 9003 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 9004   match(Set dst (EncodePKlass src));
 9005 
 9006   ins_cost(INSN_COST * 3);
 9007   format %{ "encode_klass_not_null $dst,$src" %}
 9008 
 9009   ins_encode %{
 9010     Register src_reg = as_Register($src$$reg);
 9011     Register dst_reg = as_Register($dst$$reg);
 9012     __ encode_klass_not_null(dst_reg, src_reg);
 9013   %}
 9014 
 9015    ins_pipe(ialu_reg);
 9016 %}
 9017 
 9018 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 9019   match(Set dst (DecodeNKlass src));
 9020 
 9021   ins_cost(INSN_COST * 3);
 9022   format %{ "decode_klass_not_null $dst,$src" %}
 9023 
 9024   ins_encode %{
 9025     Register src_reg = as_Register($src$$reg);
 9026     Register dst_reg = as_Register($dst$$reg);
 9027     if (dst_reg != src_reg) {
 9028       __ decode_klass_not_null(dst_reg, src_reg);
 9029     } else {
 9030       __ decode_klass_not_null(dst_reg);
 9031     }
 9032   %}
 9033 
 9034    ins_pipe(ialu_reg);
 9035 %}
 9036 
 9037 instruct checkCastPP(iRegPNoSp dst)
 9038 %{
 9039   match(Set dst (CheckCastPP dst));
 9040 
 9041   size(0);
 9042   format %{ "# checkcastPP of $dst" %}
 9043   ins_encode(/* empty encoding */);
 9044   ins_pipe(pipe_class_empty);
 9045 %}
 9046 
 9047 instruct castPP(iRegPNoSp dst)
 9048 %{
 9049   match(Set dst (CastPP dst));
 9050 
 9051   size(0);
 9052   format %{ "# castPP of $dst" %}
 9053   ins_encode(/* empty encoding */);
 9054   ins_pipe(pipe_class_empty);
 9055 %}
 9056 
 9057 instruct castII(iRegI dst)
 9058 %{
 9059   match(Set dst (CastII dst));
 9060 
 9061   size(0);
 9062   format %{ "# castII of $dst" %}
 9063   ins_encode(/* empty encoding */);
 9064   ins_cost(0);
 9065   ins_pipe(pipe_class_empty);
 9066 %}
 9067 
 9068 instruct castLL(iRegL dst)
 9069 %{
 9070   match(Set dst (CastLL dst));
 9071 
 9072   size(0);
 9073   format %{ "# castLL of $dst" %}
 9074   ins_encode(/* empty encoding */);
 9075   ins_cost(0);
 9076   ins_pipe(pipe_class_empty);
 9077 %}
 9078 
 9079 instruct castFF(vRegF dst)
 9080 %{
 9081   match(Set dst (CastFF dst));
 9082 
 9083   size(0);
 9084   format %{ "# castFF of $dst" %}
 9085   ins_encode(/* empty encoding */);
 9086   ins_cost(0);
 9087   ins_pipe(pipe_class_empty);
 9088 %}
 9089 
 9090 instruct castDD(vRegD dst)
 9091 %{
 9092   match(Set dst (CastDD dst));
 9093 
 9094   size(0);
 9095   format %{ "# castDD of $dst" %}
 9096   ins_encode(/* empty encoding */);
 9097   ins_cost(0);
 9098   ins_pipe(pipe_class_empty);
 9099 %}
 9100 
 9101 instruct castVVD(vecD dst)
 9102 %{
 9103   match(Set dst (CastVV dst));
 9104 
 9105   size(0);
 9106   format %{ "# castVV of $dst" %}
 9107   ins_encode(/* empty encoding */);
 9108   ins_cost(0);
 9109   ins_pipe(pipe_class_empty);
 9110 %}
 9111 
 9112 instruct castVVX(vecX dst)
 9113 %{
 9114   match(Set dst (CastVV dst));
 9115 
 9116   size(0);
 9117   format %{ "# castVV of $dst" %}
 9118   ins_encode(/* empty encoding */);
 9119   ins_cost(0);
 9120   ins_pipe(pipe_class_empty);
 9121 %}
 9122 
 9123 instruct castVV(vReg dst)
 9124 %{
 9125   match(Set dst (CastVV dst));
 9126 
 9127   size(0);
 9128   format %{ "# castVV of $dst" %}
 9129   ins_encode(/* empty encoding */);
 9130   ins_cost(0);
 9131   ins_pipe(pipe_class_empty);
 9132 %}
 9133 
 9134 instruct castVVMask(pRegGov dst)
 9135 %{
 9136   match(Set dst (CastVV dst));
 9137 
 9138   size(0);
 9139   format %{ "# castVV of $dst" %}
 9140   ins_encode(/* empty encoding */);
 9141   ins_cost(0);
 9142   ins_pipe(pipe_class_empty);
 9143 %}
 9144 
 9145 // ============================================================================
 9146 // Atomic operation instructions
 9147 //
 9148 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9149 // Store{PIL}Conditional instructions using a normal load for the
 9150 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9151 //
 9152 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9153 // pair to lock object allocations from Eden space when not using
 9154 // TLABs.
 9155 //
 9156 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9157 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9158 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9159 // only for 64-bit.
 9160 //
 9161 // We implement LoadPLocked and StorePLocked instructions using,
 9162 // respectively the AArch64 hw load-exclusive and store-conditional
 9163 // instructions. Whereas we must implement each of
 9164 // Store{IL}Conditional using a CAS which employs a pair of
 9165 // instructions comprising a load-exclusive followed by a
 9166 // store-conditional.
 9167 
 9168 
 9169 // Locked-load (linked load) of the current heap-top
 9170 // used when updating the eden heap top
 9171 // implemented using ldaxr on AArch64
 9172 
 9173 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9174 %{
 9175   match(Set dst (LoadPLocked mem));
 9176 
 9177   ins_cost(VOLATILE_REF_COST);
 9178 
 9179   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9180 
 9181   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9182 
 9183   ins_pipe(pipe_serial);
 9184 %}
 9185 
 9186 // Conditional-store of the updated heap-top.
 9187 // Used during allocation of the shared heap.
 9188 // Sets flag (EQ) on success.
 9189 // implemented using stlxr on AArch64.
 9190 
 9191 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9192 %{
 9193   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9194 
 9195   ins_cost(VOLATILE_REF_COST);
 9196 
 9197  // TODO
 9198  // do we need to do a store-conditional release or can we just use a
 9199  // plain store-conditional?
 9200 
 9201   format %{
 9202     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9203     "cmpw rscratch1, zr\t# EQ on successful write"
 9204   %}
 9205 
 9206   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9207 
 9208   ins_pipe(pipe_serial);
 9209 %}
 9210 
 9211 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9212 %{
 9213   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9214 
 9215   ins_cost(VOLATILE_REF_COST);
 9216 
 9217   format %{
 9218     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9219     "cmpw rscratch1, zr\t# EQ on successful write"
 9220   %}
 9221 
 9222   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9223 
 9224   ins_pipe(pipe_slow);
 9225 %}
 9226 
 9227 // storeIConditional also has acquire semantics, for no better reason
 9228 // than matching storeLConditional.  At the time of writing this
 9229 // comment storeIConditional was not used anywhere by AArch64.
 9230 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9231 %{
 9232   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9233 
 9234   ins_cost(VOLATILE_REF_COST);
 9235 
 9236   format %{
 9237     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9238     "cmpw rscratch1, zr\t# EQ on successful write"
 9239   %}
 9240 
 9241   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9242 
 9243   ins_pipe(pipe_slow);
 9244 %}
 9245 
 9246 // standard CompareAndSwapX when we are using barriers
 9247 // these have higher priority than the rules selected by a predicate
 9248 
 9249 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9250 // can't match them
 9251 
 9252 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9253 
 9254   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9255   ins_cost(2 * VOLATILE_REF_COST);
 9256 
 9257   effect(KILL cr);
 9258 
 9259   format %{
 9260     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9261     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9262   %}
 9263 
 9264   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9265             aarch64_enc_cset_eq(res));
 9266 
 9267   ins_pipe(pipe_slow);
 9268 %}
 9269 
 9270 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9271 
 9272   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9273   ins_cost(2 * VOLATILE_REF_COST);
 9274 
 9275   effect(KILL cr);
 9276 
 9277   format %{
 9278     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9279     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9280   %}
 9281 
 9282   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9283             aarch64_enc_cset_eq(res));
 9284 
 9285   ins_pipe(pipe_slow);
 9286 %}
 9287 
 9288 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9289 
 9290   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9291   ins_cost(2 * VOLATILE_REF_COST);
 9292 
 9293   effect(KILL cr);
 9294 
 9295  format %{
 9296     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9297     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9298  %}
 9299 
 9300  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9301             aarch64_enc_cset_eq(res));
 9302 
 9303   ins_pipe(pipe_slow);
 9304 %}
 9305 
 9306 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9307 
 9308   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9309   ins_cost(2 * VOLATILE_REF_COST);
 9310 
 9311   effect(KILL cr);
 9312 
 9313  format %{
 9314     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9315     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9316  %}
 9317 
 9318  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9319             aarch64_enc_cset_eq(res));
 9320 
 9321   ins_pipe(pipe_slow);
 9322 %}
 9323 
 9324 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9325 
 9326   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9327   predicate(n->as_LoadStore()->barrier_data() == 0);
 9328   ins_cost(2 * VOLATILE_REF_COST);
 9329 
 9330   effect(KILL cr);
 9331 
 9332  format %{
 9333     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9334     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9335  %}
 9336 
 9337  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9338             aarch64_enc_cset_eq(res));
 9339 
 9340   ins_pipe(pipe_slow);
 9341 %}
 9342 
 9343 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9344 
 9345   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9346   ins_cost(2 * VOLATILE_REF_COST);
 9347 
 9348   effect(KILL cr);
 9349 
 9350  format %{
 9351     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9352     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9353  %}
 9354 
 9355  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9356             aarch64_enc_cset_eq(res));
 9357 
 9358   ins_pipe(pipe_slow);
 9359 %}
 9360 
 9361 // alternative CompareAndSwapX when we are eliding barriers
 9362 
 9363 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9364 
 9365   predicate(needs_acquiring_load_exclusive(n));
 9366   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9367   ins_cost(VOLATILE_REF_COST);
 9368 
 9369   effect(KILL cr);
 9370 
 9371   format %{
 9372     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9373     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9374   %}
 9375 
 9376   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9377             aarch64_enc_cset_eq(res));
 9378 
 9379   ins_pipe(pipe_slow);
 9380 %}
 9381 
 9382 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9383 
 9384   predicate(needs_acquiring_load_exclusive(n));
 9385   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9386   ins_cost(VOLATILE_REF_COST);
 9387 
 9388   effect(KILL cr);
 9389 
 9390   format %{
 9391     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9392     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9393   %}
 9394 
 9395   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9396             aarch64_enc_cset_eq(res));
 9397 
 9398   ins_pipe(pipe_slow);
 9399 %}
 9400 
 9401 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9402 
 9403   predicate(needs_acquiring_load_exclusive(n));
 9404   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9405   ins_cost(VOLATILE_REF_COST);
 9406 
 9407   effect(KILL cr);
 9408 
 9409  format %{
 9410     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9411     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9412  %}
 9413 
 9414  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9415             aarch64_enc_cset_eq(res));
 9416 
 9417   ins_pipe(pipe_slow);
 9418 %}
 9419 
 9420 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9421 
 9422   predicate(needs_acquiring_load_exclusive(n));
 9423   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9424   ins_cost(VOLATILE_REF_COST);
 9425 
 9426   effect(KILL cr);
 9427 
 9428  format %{
 9429     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9430     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9431  %}
 9432 
 9433  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9434             aarch64_enc_cset_eq(res));
 9435 
 9436   ins_pipe(pipe_slow);
 9437 %}
 9438 
 9439 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9440 
 9441   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9442   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9443   ins_cost(VOLATILE_REF_COST);
 9444 
 9445   effect(KILL cr);
 9446 
 9447  format %{
 9448     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9449     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9450  %}
 9451 
 9452  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9453             aarch64_enc_cset_eq(res));
 9454 
 9455   ins_pipe(pipe_slow);
 9456 %}
 9457 
 9458 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9459 
 9460   predicate(needs_acquiring_load_exclusive(n));
 9461   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9462   ins_cost(VOLATILE_REF_COST);
 9463 
 9464   effect(KILL cr);
 9465 
 9466  format %{
 9467     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9468     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9469  %}
 9470 
 9471  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9472             aarch64_enc_cset_eq(res));
 9473 
 9474   ins_pipe(pipe_slow);
 9475 %}
 9476 
 9477 
 9478 // ---------------------------------------------------------------------
 9479 
 9480 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9481 
 9482 // Sundry CAS operations.  Note that release is always true,
 9483 // regardless of the memory ordering of the CAS.  This is because we
 9484 // need the volatile case to be sequentially consistent but there is
 9485 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9486 // can't check the type of memory ordering here, so we always emit a
 9487 // STLXR.
 9488 
 9489 // This section is generated from aarch64_ad_cas.m4
 9490 
 9491 
 9492 
 9493 // This pattern is generated automatically from cas.m4.
 9494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9495 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9496 
 9497   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9498   ins_cost(2 * VOLATILE_REF_COST);
 9499   effect(TEMP_DEF res, KILL cr);
 9500   format %{
 9501     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9502   %}
 9503   ins_encode %{
 9504     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9505                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9506                /*weak*/ false, $res$$Register);
 9507     __ sxtbw($res$$Register, $res$$Register);
 9508   %}
 9509   ins_pipe(pipe_slow);
 9510 %}
 9511 
 9512 // This pattern is generated automatically from cas.m4.
 9513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9514 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9515 
 9516   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9517   ins_cost(2 * VOLATILE_REF_COST);
 9518   effect(TEMP_DEF res, KILL cr);
 9519   format %{
 9520     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9521   %}
 9522   ins_encode %{
 9523     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9524                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9525                /*weak*/ false, $res$$Register);
 9526     __ sxthw($res$$Register, $res$$Register);
 9527   %}
 9528   ins_pipe(pipe_slow);
 9529 %}
 9530 
 9531 // This pattern is generated automatically from cas.m4.
 9532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9533 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9534 
 9535   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9536   ins_cost(2 * VOLATILE_REF_COST);
 9537   effect(TEMP_DEF res, KILL cr);
 9538   format %{
 9539     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9540   %}
 9541   ins_encode %{
 9542     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9543                Assembler::word, /*acquire*/ false, /*release*/ true,
 9544                /*weak*/ false, $res$$Register);
 9545   %}
 9546   ins_pipe(pipe_slow);
 9547 %}
 9548 
 9549 // This pattern is generated automatically from cas.m4.
 9550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9551 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9552 
 9553   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9554   ins_cost(2 * VOLATILE_REF_COST);
 9555   effect(TEMP_DEF res, KILL cr);
 9556   format %{
 9557     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9558   %}
 9559   ins_encode %{
 9560     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9561                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9562                /*weak*/ false, $res$$Register);
 9563   %}
 9564   ins_pipe(pipe_slow);
 9565 %}
 9566 
 9567 // This pattern is generated automatically from cas.m4.
 9568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9569 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9570 
 9571   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9572   ins_cost(2 * VOLATILE_REF_COST);
 9573   effect(TEMP_DEF res, KILL cr);
 9574   format %{
 9575     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9576   %}
 9577   ins_encode %{
 9578     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9579                Assembler::word, /*acquire*/ false, /*release*/ true,
 9580                /*weak*/ false, $res$$Register);
 9581   %}
 9582   ins_pipe(pipe_slow);
 9583 %}
 9584 
 9585 // This pattern is generated automatically from cas.m4.
 9586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9587 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9588   predicate(n->as_LoadStore()->barrier_data() == 0);
 9589   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9590   ins_cost(2 * VOLATILE_REF_COST);
 9591   effect(TEMP_DEF res, KILL cr);
 9592   format %{
 9593     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9594   %}
 9595   ins_encode %{
 9596     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9597                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9598                /*weak*/ false, $res$$Register);
 9599   %}
 9600   ins_pipe(pipe_slow);
 9601 %}
 9602 
 9603 // This pattern is generated automatically from cas.m4.
 9604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9605 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9606   predicate(needs_acquiring_load_exclusive(n));
 9607   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9608   ins_cost(VOLATILE_REF_COST);
 9609   effect(TEMP_DEF res, KILL cr);
 9610   format %{
 9611     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9612   %}
 9613   ins_encode %{
 9614     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9615                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9616                /*weak*/ false, $res$$Register);
 9617     __ sxtbw($res$$Register, $res$$Register);
 9618   %}
 9619   ins_pipe(pipe_slow);
 9620 %}
 9621 
 9622 // This pattern is generated automatically from cas.m4.
 9623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9624 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9625   predicate(needs_acquiring_load_exclusive(n));
 9626   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9627   ins_cost(VOLATILE_REF_COST);
 9628   effect(TEMP_DEF res, KILL cr);
 9629   format %{
 9630     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9631   %}
 9632   ins_encode %{
 9633     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9634                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9635                /*weak*/ false, $res$$Register);
 9636     __ sxthw($res$$Register, $res$$Register);
 9637   %}
 9638   ins_pipe(pipe_slow);
 9639 %}
 9640 
 9641 // This pattern is generated automatically from cas.m4.
 9642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9643 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9644   predicate(needs_acquiring_load_exclusive(n));
 9645   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9646   ins_cost(VOLATILE_REF_COST);
 9647   effect(TEMP_DEF res, KILL cr);
 9648   format %{
 9649     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9650   %}
 9651   ins_encode %{
 9652     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9653                Assembler::word, /*acquire*/ true, /*release*/ true,
 9654                /*weak*/ false, $res$$Register);
 9655   %}
 9656   ins_pipe(pipe_slow);
 9657 %}
 9658 
 9659 // This pattern is generated automatically from cas.m4.
 9660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9661 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9662   predicate(needs_acquiring_load_exclusive(n));
 9663   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9664   ins_cost(VOLATILE_REF_COST);
 9665   effect(TEMP_DEF res, KILL cr);
 9666   format %{
 9667     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9668   %}
 9669   ins_encode %{
 9670     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9671                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9672                /*weak*/ false, $res$$Register);
 9673   %}
 9674   ins_pipe(pipe_slow);
 9675 %}
 9676 
 9677 // This pattern is generated automatically from cas.m4.
 9678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9679 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9680   predicate(needs_acquiring_load_exclusive(n));
 9681   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9682   ins_cost(VOLATILE_REF_COST);
 9683   effect(TEMP_DEF res, KILL cr);
 9684   format %{
 9685     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9686   %}
 9687   ins_encode %{
 9688     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9689                Assembler::word, /*acquire*/ true, /*release*/ true,
 9690                /*weak*/ false, $res$$Register);
 9691   %}
 9692   ins_pipe(pipe_slow);
 9693 %}
 9694 
 9695 // This pattern is generated automatically from cas.m4.
 9696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9697 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9698   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9699   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9700   ins_cost(VOLATILE_REF_COST);
 9701   effect(TEMP_DEF res, KILL cr);
 9702   format %{
 9703     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9704   %}
 9705   ins_encode %{
 9706     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9707                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9708                /*weak*/ false, $res$$Register);
 9709   %}
 9710   ins_pipe(pipe_slow);
 9711 %}
 9712 
 9713 // This pattern is generated automatically from cas.m4.
 9714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9715 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9716 
 9717   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9718   ins_cost(2 * VOLATILE_REF_COST);
 9719   effect(KILL cr);
 9720   format %{
 9721     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9722     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9723   %}
 9724   ins_encode %{
 9725     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9726                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9727                /*weak*/ true, noreg);
 9728     __ csetw($res$$Register, Assembler::EQ);
 9729   %}
 9730   ins_pipe(pipe_slow);
 9731 %}
 9732 
 9733 // This pattern is generated automatically from cas.m4.
 9734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9735 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9736 
 9737   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9738   ins_cost(2 * VOLATILE_REF_COST);
 9739   effect(KILL cr);
 9740   format %{
 9741     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9742     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9743   %}
 9744   ins_encode %{
 9745     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9746                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9747                /*weak*/ true, noreg);
 9748     __ csetw($res$$Register, Assembler::EQ);
 9749   %}
 9750   ins_pipe(pipe_slow);
 9751 %}
 9752 
 9753 // This pattern is generated automatically from cas.m4.
 9754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9755 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9756 
 9757   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9758   ins_cost(2 * VOLATILE_REF_COST);
 9759   effect(KILL cr);
 9760   format %{
 9761     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9762     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9763   %}
 9764   ins_encode %{
 9765     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9766                Assembler::word, /*acquire*/ false, /*release*/ true,
 9767                /*weak*/ true, noreg);
 9768     __ csetw($res$$Register, Assembler::EQ);
 9769   %}
 9770   ins_pipe(pipe_slow);
 9771 %}
 9772 
 9773 // This pattern is generated automatically from cas.m4.
 9774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9775 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9776 
 9777   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9778   ins_cost(2 * VOLATILE_REF_COST);
 9779   effect(KILL cr);
 9780   format %{
 9781     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9782     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9783   %}
 9784   ins_encode %{
 9785     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9786                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9787                /*weak*/ true, noreg);
 9788     __ csetw($res$$Register, Assembler::EQ);
 9789   %}
 9790   ins_pipe(pipe_slow);
 9791 %}
 9792 
 9793 // This pattern is generated automatically from cas.m4.
 9794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9795 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9796 
 9797   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9798   ins_cost(2 * VOLATILE_REF_COST);
 9799   effect(KILL cr);
 9800   format %{
 9801     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9802     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9803   %}
 9804   ins_encode %{
 9805     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9806                Assembler::word, /*acquire*/ false, /*release*/ true,
 9807                /*weak*/ true, noreg);
 9808     __ csetw($res$$Register, Assembler::EQ);
 9809   %}
 9810   ins_pipe(pipe_slow);
 9811 %}
 9812 
 9813 // This pattern is generated automatically from cas.m4.
 9814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9815 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9816   predicate(n->as_LoadStore()->barrier_data() == 0);
 9817   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9818   ins_cost(2 * VOLATILE_REF_COST);
 9819   effect(KILL cr);
 9820   format %{
 9821     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9822     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9823   %}
 9824   ins_encode %{
 9825     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9826                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9827                /*weak*/ true, noreg);
 9828     __ csetw($res$$Register, Assembler::EQ);
 9829   %}
 9830   ins_pipe(pipe_slow);
 9831 %}
 9832 
 9833 // This pattern is generated automatically from cas.m4.
 9834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9835 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9836   predicate(needs_acquiring_load_exclusive(n));
 9837   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9838   ins_cost(VOLATILE_REF_COST);
 9839   effect(KILL cr);
 9840   format %{
 9841     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9842     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9843   %}
 9844   ins_encode %{
 9845     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9846                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9847                /*weak*/ true, noreg);
 9848     __ csetw($res$$Register, Assembler::EQ);
 9849   %}
 9850   ins_pipe(pipe_slow);
 9851 %}
 9852 
 9853 // This pattern is generated automatically from cas.m4.
 9854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9855 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9856   predicate(needs_acquiring_load_exclusive(n));
 9857   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9858   ins_cost(VOLATILE_REF_COST);
 9859   effect(KILL cr);
 9860   format %{
 9861     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9862     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9863   %}
 9864   ins_encode %{
 9865     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9866                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9867                /*weak*/ true, noreg);
 9868     __ csetw($res$$Register, Assembler::EQ);
 9869   %}
 9870   ins_pipe(pipe_slow);
 9871 %}
 9872 
 9873 // This pattern is generated automatically from cas.m4.
 9874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9875 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9876   predicate(needs_acquiring_load_exclusive(n));
 9877   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9878   ins_cost(VOLATILE_REF_COST);
 9879   effect(KILL cr);
 9880   format %{
 9881     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9882     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9883   %}
 9884   ins_encode %{
 9885     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9886                Assembler::word, /*acquire*/ true, /*release*/ true,
 9887                /*weak*/ true, noreg);
 9888     __ csetw($res$$Register, Assembler::EQ);
 9889   %}
 9890   ins_pipe(pipe_slow);
 9891 %}
 9892 
 9893 // This pattern is generated automatically from cas.m4.
 9894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9895 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9896   predicate(needs_acquiring_load_exclusive(n));
 9897   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9898   ins_cost(VOLATILE_REF_COST);
 9899   effect(KILL cr);
 9900   format %{
 9901     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9902     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9903   %}
 9904   ins_encode %{
 9905     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9906                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9907                /*weak*/ true, noreg);
 9908     __ csetw($res$$Register, Assembler::EQ);
 9909   %}
 9910   ins_pipe(pipe_slow);
 9911 %}
 9912 
 9913 // This pattern is generated automatically from cas.m4.
 9914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9915 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9916   predicate(needs_acquiring_load_exclusive(n));
 9917   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9918   ins_cost(VOLATILE_REF_COST);
 9919   effect(KILL cr);
 9920   format %{
 9921     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9922     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9923   %}
 9924   ins_encode %{
 9925     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9926                Assembler::word, /*acquire*/ true, /*release*/ true,
 9927                /*weak*/ true, noreg);
 9928     __ csetw($res$$Register, Assembler::EQ);
 9929   %}
 9930   ins_pipe(pipe_slow);
 9931 %}
 9932 
 9933 // This pattern is generated automatically from cas.m4.
 9934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9935 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9936   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9937   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9938   ins_cost(VOLATILE_REF_COST);
 9939   effect(KILL cr);
 9940   format %{
 9941     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9942     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9943   %}
 9944   ins_encode %{
 9945     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9946                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9947                /*weak*/ true, noreg);
 9948     __ csetw($res$$Register, Assembler::EQ);
 9949   %}
 9950   ins_pipe(pipe_slow);
 9951 %}
 9952 
 9953 // END This section of the file is automatically generated. Do not edit --------------
 9954 // ---------------------------------------------------------------------
 9955 
 9956 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9957   match(Set prev (GetAndSetI mem newv));
 9958   ins_cost(2 * VOLATILE_REF_COST);
 9959   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9960   ins_encode %{
 9961     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9962   %}
 9963   ins_pipe(pipe_serial);
 9964 %}
 9965 
 9966 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9967   match(Set prev (GetAndSetL mem newv));
 9968   ins_cost(2 * VOLATILE_REF_COST);
 9969   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9970   ins_encode %{
 9971     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9972   %}
 9973   ins_pipe(pipe_serial);
 9974 %}
 9975 
 9976 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9977   match(Set prev (GetAndSetN mem newv));
 9978   ins_cost(2 * VOLATILE_REF_COST);
 9979   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9980   ins_encode %{
 9981     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9982   %}
 9983   ins_pipe(pipe_serial);
 9984 %}
 9985 
 9986 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9987   predicate(n->as_LoadStore()->barrier_data() == 0);
 9988   match(Set prev (GetAndSetP mem newv));
 9989   ins_cost(2 * VOLATILE_REF_COST);
 9990   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9991   ins_encode %{
 9992     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9993   %}
 9994   ins_pipe(pipe_serial);
 9995 %}
 9996 
 9997 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9998   predicate(needs_acquiring_load_exclusive(n));
 9999   match(Set prev (GetAndSetI mem newv));
10000   ins_cost(VOLATILE_REF_COST);
10001   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
10002   ins_encode %{
10003     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10004   %}
10005   ins_pipe(pipe_serial);
10006 %}
10007 
10008 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
10009   predicate(needs_acquiring_load_exclusive(n));
10010   match(Set prev (GetAndSetL mem newv));
10011   ins_cost(VOLATILE_REF_COST);
10012   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
10013   ins_encode %{
10014     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10015   %}
10016   ins_pipe(pipe_serial);
10017 %}
10018 
10019 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
10020   predicate(needs_acquiring_load_exclusive(n));
10021   match(Set prev (GetAndSetN mem newv));
10022   ins_cost(VOLATILE_REF_COST);
10023   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
10024   ins_encode %{
10025     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10026   %}
10027   ins_pipe(pipe_serial);
10028 %}
10029 
10030 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
10031   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
10032   match(Set prev (GetAndSetP mem newv));
10033   ins_cost(VOLATILE_REF_COST);
10034   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
10035   ins_encode %{
10036     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10037   %}
10038   ins_pipe(pipe_serial);
10039 %}
10040 
10041 
10042 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
10043   match(Set newval (GetAndAddL mem incr));
10044   ins_cost(2 * VOLATILE_REF_COST + 1);
10045   format %{ "get_and_addL $newval, [$mem], $incr" %}
10046   ins_encode %{
10047     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
10048   %}
10049   ins_pipe(pipe_serial);
10050 %}
10051 
10052 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
10053   predicate(n->as_LoadStore()->result_not_used());
10054   match(Set dummy (GetAndAddL mem incr));
10055   ins_cost(2 * VOLATILE_REF_COST);
10056   format %{ "get_and_addL [$mem], $incr" %}
10057   ins_encode %{
10058     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
10059   %}
10060   ins_pipe(pipe_serial);
10061 %}
10062 
10063 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10064   match(Set newval (GetAndAddL mem incr));
10065   ins_cost(2 * VOLATILE_REF_COST + 1);
10066   format %{ "get_and_addL $newval, [$mem], $incr" %}
10067   ins_encode %{
10068     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
10069   %}
10070   ins_pipe(pipe_serial);
10071 %}
10072 
10073 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
10074   predicate(n->as_LoadStore()->result_not_used());
10075   match(Set dummy (GetAndAddL mem incr));
10076   ins_cost(2 * VOLATILE_REF_COST);
10077   format %{ "get_and_addL [$mem], $incr" %}
10078   ins_encode %{
10079     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
10080   %}
10081   ins_pipe(pipe_serial);
10082 %}
10083 
10084 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10085   match(Set newval (GetAndAddI mem incr));
10086   ins_cost(2 * VOLATILE_REF_COST + 1);
10087   format %{ "get_and_addI $newval, [$mem], $incr" %}
10088   ins_encode %{
10089     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10090   %}
10091   ins_pipe(pipe_serial);
10092 %}
10093 
10094 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10095   predicate(n->as_LoadStore()->result_not_used());
10096   match(Set dummy (GetAndAddI mem incr));
10097   ins_cost(2 * VOLATILE_REF_COST);
10098   format %{ "get_and_addI [$mem], $incr" %}
10099   ins_encode %{
10100     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10101   %}
10102   ins_pipe(pipe_serial);
10103 %}
10104 
10105 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10106   match(Set newval (GetAndAddI mem incr));
10107   ins_cost(2 * VOLATILE_REF_COST + 1);
10108   format %{ "get_and_addI $newval, [$mem], $incr" %}
10109   ins_encode %{
10110     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10111   %}
10112   ins_pipe(pipe_serial);
10113 %}
10114 
10115 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10116   predicate(n->as_LoadStore()->result_not_used());
10117   match(Set dummy (GetAndAddI mem incr));
10118   ins_cost(2 * VOLATILE_REF_COST);
10119   format %{ "get_and_addI [$mem], $incr" %}
10120   ins_encode %{
10121     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10122   %}
10123   ins_pipe(pipe_serial);
10124 %}
10125 
10126 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10127   predicate(needs_acquiring_load_exclusive(n));
10128   match(Set newval (GetAndAddL mem incr));
10129   ins_cost(VOLATILE_REF_COST + 1);
10130   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10131   ins_encode %{
10132     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10133   %}
10134   ins_pipe(pipe_serial);
10135 %}
10136 
10137 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10138   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10139   match(Set dummy (GetAndAddL mem incr));
10140   ins_cost(VOLATILE_REF_COST);
10141   format %{ "get_and_addL_acq [$mem], $incr" %}
10142   ins_encode %{
10143     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10144   %}
10145   ins_pipe(pipe_serial);
10146 %}
10147 
10148 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10149   predicate(needs_acquiring_load_exclusive(n));
10150   match(Set newval (GetAndAddL mem incr));
10151   ins_cost(VOLATILE_REF_COST + 1);
10152   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10153   ins_encode %{
10154     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10155   %}
10156   ins_pipe(pipe_serial);
10157 %}
10158 
10159 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10160   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10161   match(Set dummy (GetAndAddL mem incr));
10162   ins_cost(VOLATILE_REF_COST);
10163   format %{ "get_and_addL_acq [$mem], $incr" %}
10164   ins_encode %{
10165     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10166   %}
10167   ins_pipe(pipe_serial);
10168 %}
10169 
10170 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10171   predicate(needs_acquiring_load_exclusive(n));
10172   match(Set newval (GetAndAddI mem incr));
10173   ins_cost(VOLATILE_REF_COST + 1);
10174   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10175   ins_encode %{
10176     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10177   %}
10178   ins_pipe(pipe_serial);
10179 %}
10180 
10181 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10182   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10183   match(Set dummy (GetAndAddI mem incr));
10184   ins_cost(VOLATILE_REF_COST);
10185   format %{ "get_and_addI_acq [$mem], $incr" %}
10186   ins_encode %{
10187     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10188   %}
10189   ins_pipe(pipe_serial);
10190 %}
10191 
10192 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10193   predicate(needs_acquiring_load_exclusive(n));
10194   match(Set newval (GetAndAddI mem incr));
10195   ins_cost(VOLATILE_REF_COST + 1);
10196   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10197   ins_encode %{
10198     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10199   %}
10200   ins_pipe(pipe_serial);
10201 %}
10202 
10203 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10204   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10205   match(Set dummy (GetAndAddI mem incr));
10206   ins_cost(VOLATILE_REF_COST);
10207   format %{ "get_and_addI_acq [$mem], $incr" %}
10208   ins_encode %{
10209     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10210   %}
10211   ins_pipe(pipe_serial);
10212 %}
10213 
10214 // Manifest a CmpL result in an integer register.
10215 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10216 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10217 %{
10218   match(Set dst (CmpL3 src1 src2));
10219   effect(KILL flags);
10220 
10221   ins_cost(INSN_COST * 6);
10222   format %{
10223       "cmp $src1, $src2"
10224       "csetw $dst, ne"
10225       "cnegw $dst, lt"
10226   %}
10227   // format %{ "CmpL3 $dst, $src1, $src2" %}
10228   ins_encode %{
10229     __ cmp($src1$$Register, $src2$$Register);
10230     __ csetw($dst$$Register, Assembler::NE);
10231     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10232   %}
10233 
10234   ins_pipe(pipe_class_default);
10235 %}
10236 
10237 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10238 %{
10239   match(Set dst (CmpL3 src1 src2));
10240   effect(KILL flags);
10241 
10242   ins_cost(INSN_COST * 6);
10243   format %{
10244       "cmp $src1, $src2"
10245       "csetw $dst, ne"
10246       "cnegw $dst, lt"
10247   %}
10248   ins_encode %{
10249     int32_t con = (int32_t)$src2$$constant;
10250      if (con < 0) {
10251       __ adds(zr, $src1$$Register, -con);
10252     } else {
10253       __ subs(zr, $src1$$Register, con);
10254     }
10255     __ csetw($dst$$Register, Assembler::NE);
10256     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10257   %}
10258 
10259   ins_pipe(pipe_class_default);
10260 %}
10261 
10262 // ============================================================================
10263 // Conditional Move Instructions
10264 
10265 // n.b. we have identical rules for both a signed compare op (cmpOp)
10266 // and an unsigned compare op (cmpOpU). it would be nice if we could
10267 // define an op class which merged both inputs and use it to type the
10268 // argument to a single rule. unfortunatelyt his fails because the
10269 // opclass does not live up to the COND_INTER interface of its
10270 // component operands. When the generic code tries to negate the
10271 // operand it ends up running the generci Machoper::negate method
10272 // which throws a ShouldNotHappen. So, we have to provide two flavours
10273 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10274 
10275 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10276   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10277 
10278   ins_cost(INSN_COST * 2);
10279   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10280 
10281   ins_encode %{
10282     __ cselw(as_Register($dst$$reg),
10283              as_Register($src2$$reg),
10284              as_Register($src1$$reg),
10285              (Assembler::Condition)$cmp$$cmpcode);
10286   %}
10287 
10288   ins_pipe(icond_reg_reg);
10289 %}
10290 
10291 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10292   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10293 
10294   ins_cost(INSN_COST * 2);
10295   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10296 
10297   ins_encode %{
10298     __ cselw(as_Register($dst$$reg),
10299              as_Register($src2$$reg),
10300              as_Register($src1$$reg),
10301              (Assembler::Condition)$cmp$$cmpcode);
10302   %}
10303 
10304   ins_pipe(icond_reg_reg);
10305 %}
10306 
10307 // special cases where one arg is zero
10308 
10309 // n.b. this is selected in preference to the rule above because it
10310 // avoids loading constant 0 into a source register
10311 
10312 // TODO
10313 // we ought only to be able to cull one of these variants as the ideal
10314 // transforms ought always to order the zero consistently (to left/right?)
10315 
10316 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10317   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10318 
10319   ins_cost(INSN_COST * 2);
10320   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10321 
10322   ins_encode %{
10323     __ cselw(as_Register($dst$$reg),
10324              as_Register($src$$reg),
10325              zr,
10326              (Assembler::Condition)$cmp$$cmpcode);
10327   %}
10328 
10329   ins_pipe(icond_reg);
10330 %}
10331 
10332 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10333   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10334 
10335   ins_cost(INSN_COST * 2);
10336   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10337 
10338   ins_encode %{
10339     __ cselw(as_Register($dst$$reg),
10340              as_Register($src$$reg),
10341              zr,
10342              (Assembler::Condition)$cmp$$cmpcode);
10343   %}
10344 
10345   ins_pipe(icond_reg);
10346 %}
10347 
10348 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10349   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10350 
10351   ins_cost(INSN_COST * 2);
10352   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10353 
10354   ins_encode %{
10355     __ cselw(as_Register($dst$$reg),
10356              zr,
10357              as_Register($src$$reg),
10358              (Assembler::Condition)$cmp$$cmpcode);
10359   %}
10360 
10361   ins_pipe(icond_reg);
10362 %}
10363 
10364 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10365   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10366 
10367   ins_cost(INSN_COST * 2);
10368   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10369 
10370   ins_encode %{
10371     __ cselw(as_Register($dst$$reg),
10372              zr,
10373              as_Register($src$$reg),
10374              (Assembler::Condition)$cmp$$cmpcode);
10375   %}
10376 
10377   ins_pipe(icond_reg);
10378 %}
10379 
10380 // special case for creating a boolean 0 or 1
10381 
10382 // n.b. this is selected in preference to the rule above because it
10383 // avoids loading constants 0 and 1 into a source register
10384 
10385 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10386   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10387 
10388   ins_cost(INSN_COST * 2);
10389   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10390 
10391   ins_encode %{
10392     // equivalently
10393     // cset(as_Register($dst$$reg),
10394     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10395     __ csincw(as_Register($dst$$reg),
10396              zr,
10397              zr,
10398              (Assembler::Condition)$cmp$$cmpcode);
10399   %}
10400 
10401   ins_pipe(icond_none);
10402 %}
10403 
10404 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10405   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10406 
10407   ins_cost(INSN_COST * 2);
10408   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10409 
10410   ins_encode %{
10411     // equivalently
10412     // cset(as_Register($dst$$reg),
10413     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10414     __ csincw(as_Register($dst$$reg),
10415              zr,
10416              zr,
10417              (Assembler::Condition)$cmp$$cmpcode);
10418   %}
10419 
10420   ins_pipe(icond_none);
10421 %}
10422 
10423 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10424   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10425 
10426   ins_cost(INSN_COST * 2);
10427   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10428 
10429   ins_encode %{
10430     __ csel(as_Register($dst$$reg),
10431             as_Register($src2$$reg),
10432             as_Register($src1$$reg),
10433             (Assembler::Condition)$cmp$$cmpcode);
10434   %}
10435 
10436   ins_pipe(icond_reg_reg);
10437 %}
10438 
10439 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10440   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10441 
10442   ins_cost(INSN_COST * 2);
10443   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10444 
10445   ins_encode %{
10446     __ csel(as_Register($dst$$reg),
10447             as_Register($src2$$reg),
10448             as_Register($src1$$reg),
10449             (Assembler::Condition)$cmp$$cmpcode);
10450   %}
10451 
10452   ins_pipe(icond_reg_reg);
10453 %}
10454 
10455 // special cases where one arg is zero
10456 
10457 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10458   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10459 
10460   ins_cost(INSN_COST * 2);
10461   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10462 
10463   ins_encode %{
10464     __ csel(as_Register($dst$$reg),
10465             zr,
10466             as_Register($src$$reg),
10467             (Assembler::Condition)$cmp$$cmpcode);
10468   %}
10469 
10470   ins_pipe(icond_reg);
10471 %}
10472 
10473 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10474   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10475 
10476   ins_cost(INSN_COST * 2);
10477   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10478 
10479   ins_encode %{
10480     __ csel(as_Register($dst$$reg),
10481             zr,
10482             as_Register($src$$reg),
10483             (Assembler::Condition)$cmp$$cmpcode);
10484   %}
10485 
10486   ins_pipe(icond_reg);
10487 %}
10488 
10489 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10490   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10491 
10492   ins_cost(INSN_COST * 2);
10493   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10494 
10495   ins_encode %{
10496     __ csel(as_Register($dst$$reg),
10497             as_Register($src$$reg),
10498             zr,
10499             (Assembler::Condition)$cmp$$cmpcode);
10500   %}
10501 
10502   ins_pipe(icond_reg);
10503 %}
10504 
10505 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10506   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10507 
10508   ins_cost(INSN_COST * 2);
10509   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10510 
10511   ins_encode %{
10512     __ csel(as_Register($dst$$reg),
10513             as_Register($src$$reg),
10514             zr,
10515             (Assembler::Condition)$cmp$$cmpcode);
10516   %}
10517 
10518   ins_pipe(icond_reg);
10519 %}
10520 
10521 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10522   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10523 
10524   ins_cost(INSN_COST * 2);
10525   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10526 
10527   ins_encode %{
10528     __ csel(as_Register($dst$$reg),
10529             as_Register($src2$$reg),
10530             as_Register($src1$$reg),
10531             (Assembler::Condition)$cmp$$cmpcode);
10532   %}
10533 
10534   ins_pipe(icond_reg_reg);
10535 %}
10536 
10537 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10538   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10539 
10540   ins_cost(INSN_COST * 2);
10541   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10542 
10543   ins_encode %{
10544     __ csel(as_Register($dst$$reg),
10545             as_Register($src2$$reg),
10546             as_Register($src1$$reg),
10547             (Assembler::Condition)$cmp$$cmpcode);
10548   %}
10549 
10550   ins_pipe(icond_reg_reg);
10551 %}
10552 
10553 // special cases where one arg is zero
10554 
10555 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10556   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10557 
10558   ins_cost(INSN_COST * 2);
10559   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10560 
10561   ins_encode %{
10562     __ csel(as_Register($dst$$reg),
10563             zr,
10564             as_Register($src$$reg),
10565             (Assembler::Condition)$cmp$$cmpcode);
10566   %}
10567 
10568   ins_pipe(icond_reg);
10569 %}
10570 
10571 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10572   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10573 
10574   ins_cost(INSN_COST * 2);
10575   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10576 
10577   ins_encode %{
10578     __ csel(as_Register($dst$$reg),
10579             zr,
10580             as_Register($src$$reg),
10581             (Assembler::Condition)$cmp$$cmpcode);
10582   %}
10583 
10584   ins_pipe(icond_reg);
10585 %}
10586 
10587 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10588   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10589 
10590   ins_cost(INSN_COST * 2);
10591   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10592 
10593   ins_encode %{
10594     __ csel(as_Register($dst$$reg),
10595             as_Register($src$$reg),
10596             zr,
10597             (Assembler::Condition)$cmp$$cmpcode);
10598   %}
10599 
10600   ins_pipe(icond_reg);
10601 %}
10602 
10603 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10604   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10605 
10606   ins_cost(INSN_COST * 2);
10607   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10608 
10609   ins_encode %{
10610     __ csel(as_Register($dst$$reg),
10611             as_Register($src$$reg),
10612             zr,
10613             (Assembler::Condition)$cmp$$cmpcode);
10614   %}
10615 
10616   ins_pipe(icond_reg);
10617 %}
10618 
10619 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10620   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10621 
10622   ins_cost(INSN_COST * 2);
10623   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10624 
10625   ins_encode %{
10626     __ cselw(as_Register($dst$$reg),
10627              as_Register($src2$$reg),
10628              as_Register($src1$$reg),
10629              (Assembler::Condition)$cmp$$cmpcode);
10630   %}
10631 
10632   ins_pipe(icond_reg_reg);
10633 %}
10634 
10635 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10636   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10637 
10638   ins_cost(INSN_COST * 2);
10639   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10640 
10641   ins_encode %{
10642     __ cselw(as_Register($dst$$reg),
10643              as_Register($src2$$reg),
10644              as_Register($src1$$reg),
10645              (Assembler::Condition)$cmp$$cmpcode);
10646   %}
10647 
10648   ins_pipe(icond_reg_reg);
10649 %}
10650 
10651 // special cases where one arg is zero
10652 
10653 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10654   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10655 
10656   ins_cost(INSN_COST * 2);
10657   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10658 
10659   ins_encode %{
10660     __ cselw(as_Register($dst$$reg),
10661              zr,
10662              as_Register($src$$reg),
10663              (Assembler::Condition)$cmp$$cmpcode);
10664   %}
10665 
10666   ins_pipe(icond_reg);
10667 %}
10668 
10669 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10670   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10671 
10672   ins_cost(INSN_COST * 2);
10673   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10674 
10675   ins_encode %{
10676     __ cselw(as_Register($dst$$reg),
10677              zr,
10678              as_Register($src$$reg),
10679              (Assembler::Condition)$cmp$$cmpcode);
10680   %}
10681 
10682   ins_pipe(icond_reg);
10683 %}
10684 
10685 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10686   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10687 
10688   ins_cost(INSN_COST * 2);
10689   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10690 
10691   ins_encode %{
10692     __ cselw(as_Register($dst$$reg),
10693              as_Register($src$$reg),
10694              zr,
10695              (Assembler::Condition)$cmp$$cmpcode);
10696   %}
10697 
10698   ins_pipe(icond_reg);
10699 %}
10700 
10701 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10702   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10703 
10704   ins_cost(INSN_COST * 2);
10705   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10706 
10707   ins_encode %{
10708     __ cselw(as_Register($dst$$reg),
10709              as_Register($src$$reg),
10710              zr,
10711              (Assembler::Condition)$cmp$$cmpcode);
10712   %}
10713 
10714   ins_pipe(icond_reg);
10715 %}
10716 
10717 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10718 %{
10719   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10720 
10721   ins_cost(INSN_COST * 3);
10722 
10723   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10724   ins_encode %{
10725     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10726     __ fcsels(as_FloatRegister($dst$$reg),
10727               as_FloatRegister($src2$$reg),
10728               as_FloatRegister($src1$$reg),
10729               cond);
10730   %}
10731 
10732   ins_pipe(fp_cond_reg_reg_s);
10733 %}
10734 
10735 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10736 %{
10737   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10738 
10739   ins_cost(INSN_COST * 3);
10740 
10741   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10742   ins_encode %{
10743     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10744     __ fcsels(as_FloatRegister($dst$$reg),
10745               as_FloatRegister($src2$$reg),
10746               as_FloatRegister($src1$$reg),
10747               cond);
10748   %}
10749 
10750   ins_pipe(fp_cond_reg_reg_s);
10751 %}
10752 
10753 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10754 %{
10755   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10756 
10757   ins_cost(INSN_COST * 3);
10758 
10759   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10760   ins_encode %{
10761     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10762     __ fcseld(as_FloatRegister($dst$$reg),
10763               as_FloatRegister($src2$$reg),
10764               as_FloatRegister($src1$$reg),
10765               cond);
10766   %}
10767 
10768   ins_pipe(fp_cond_reg_reg_d);
10769 %}
10770 
10771 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10772 %{
10773   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10774 
10775   ins_cost(INSN_COST * 3);
10776 
10777   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10778   ins_encode %{
10779     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10780     __ fcseld(as_FloatRegister($dst$$reg),
10781               as_FloatRegister($src2$$reg),
10782               as_FloatRegister($src1$$reg),
10783               cond);
10784   %}
10785 
10786   ins_pipe(fp_cond_reg_reg_d);
10787 %}
10788 
10789 // ============================================================================
10790 // Arithmetic Instructions
10791 //
10792 
10793 // Integer Addition
10794 
10795 // TODO
10796 // these currently employ operations which do not set CR and hence are
10797 // not flagged as killing CR but we would like to isolate the cases
10798 // where we want to set flags from those where we don't. need to work
10799 // out how to do that.
10800 
10801 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10802   match(Set dst (AddI src1 src2));
10803 
10804   ins_cost(INSN_COST);
10805   format %{ "addw  $dst, $src1, $src2" %}
10806 
10807   ins_encode %{
10808     __ addw(as_Register($dst$$reg),
10809             as_Register($src1$$reg),
10810             as_Register($src2$$reg));
10811   %}
10812 
10813   ins_pipe(ialu_reg_reg);
10814 %}
10815 
10816 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10817   match(Set dst (AddI src1 src2));
10818 
10819   ins_cost(INSN_COST);
10820   format %{ "addw $dst, $src1, $src2" %}
10821 
10822   // use opcode to indicate that this is an add not a sub
10823   opcode(0x0);
10824 
10825   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10826 
10827   ins_pipe(ialu_reg_imm);
10828 %}
10829 
10830 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10831   match(Set dst (AddI (ConvL2I src1) src2));
10832 
10833   ins_cost(INSN_COST);
10834   format %{ "addw $dst, $src1, $src2" %}
10835 
10836   // use opcode to indicate that this is an add not a sub
10837   opcode(0x0);
10838 
10839   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10840 
10841   ins_pipe(ialu_reg_imm);
10842 %}
10843 
10844 // Pointer Addition
10845 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10846   match(Set dst (AddP src1 src2));
10847 
10848   ins_cost(INSN_COST);
10849   format %{ "add $dst, $src1, $src2\t# ptr" %}
10850 
10851   ins_encode %{
10852     __ add(as_Register($dst$$reg),
10853            as_Register($src1$$reg),
10854            as_Register($src2$$reg));
10855   %}
10856 
10857   ins_pipe(ialu_reg_reg);
10858 %}
10859 
10860 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10861   match(Set dst (AddP src1 (ConvI2L src2)));
10862 
10863   ins_cost(1.9 * INSN_COST);
10864   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10865 
10866   ins_encode %{
10867     __ add(as_Register($dst$$reg),
10868            as_Register($src1$$reg),
10869            as_Register($src2$$reg), ext::sxtw);
10870   %}
10871 
10872   ins_pipe(ialu_reg_reg);
10873 %}
10874 
10875 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10876   match(Set dst (AddP src1 (LShiftL src2 scale)));
10877 
10878   ins_cost(1.9 * INSN_COST);
10879   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10880 
10881   ins_encode %{
10882     __ lea(as_Register($dst$$reg),
10883            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10884                    Address::lsl($scale$$constant)));
10885   %}
10886 
10887   ins_pipe(ialu_reg_reg_shift);
10888 %}
10889 
10890 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10891   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10892 
10893   ins_cost(1.9 * INSN_COST);
10894   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10895 
10896   ins_encode %{
10897     __ lea(as_Register($dst$$reg),
10898            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10899                    Address::sxtw($scale$$constant)));
10900   %}
10901 
10902   ins_pipe(ialu_reg_reg_shift);
10903 %}
10904 
10905 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10906   match(Set dst (LShiftL (ConvI2L src) scale));
10907 
10908   ins_cost(INSN_COST);
10909   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10910 
10911   ins_encode %{
10912     __ sbfiz(as_Register($dst$$reg),
10913           as_Register($src$$reg),
10914           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10915   %}
10916 
10917   ins_pipe(ialu_reg_shift);
10918 %}
10919 
10920 // Pointer Immediate Addition
10921 // n.b. this needs to be more expensive than using an indirect memory
10922 // operand
10923 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10924   match(Set dst (AddP src1 src2));
10925 
10926   ins_cost(INSN_COST);
10927   format %{ "add $dst, $src1, $src2\t# ptr" %}
10928 
10929   // use opcode to indicate that this is an add not a sub
10930   opcode(0x0);
10931 
10932   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10933 
10934   ins_pipe(ialu_reg_imm);
10935 %}
10936 
10937 // Long Addition
10938 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10939 
10940   match(Set dst (AddL src1 src2));
10941 
10942   ins_cost(INSN_COST);
10943   format %{ "add  $dst, $src1, $src2" %}
10944 
10945   ins_encode %{
10946     __ add(as_Register($dst$$reg),
10947            as_Register($src1$$reg),
10948            as_Register($src2$$reg));
10949   %}
10950 
10951   ins_pipe(ialu_reg_reg);
10952 %}
10953 
10954 // No constant pool entries requiredLong Immediate Addition.
10955 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10956   match(Set dst (AddL src1 src2));
10957 
10958   ins_cost(INSN_COST);
10959   format %{ "add $dst, $src1, $src2" %}
10960 
10961   // use opcode to indicate that this is an add not a sub
10962   opcode(0x0);
10963 
10964   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10965 
10966   ins_pipe(ialu_reg_imm);
10967 %}
10968 
10969 // Integer Subtraction
10970 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10971   match(Set dst (SubI src1 src2));
10972 
10973   ins_cost(INSN_COST);
10974   format %{ "subw  $dst, $src1, $src2" %}
10975 
10976   ins_encode %{
10977     __ subw(as_Register($dst$$reg),
10978             as_Register($src1$$reg),
10979             as_Register($src2$$reg));
10980   %}
10981 
10982   ins_pipe(ialu_reg_reg);
10983 %}
10984 
10985 // Immediate Subtraction
10986 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10987   match(Set dst (SubI src1 src2));
10988 
10989   ins_cost(INSN_COST);
10990   format %{ "subw $dst, $src1, $src2" %}
10991 
10992   // use opcode to indicate that this is a sub not an add
10993   opcode(0x1);
10994 
10995   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10996 
10997   ins_pipe(ialu_reg_imm);
10998 %}
10999 
11000 // Long Subtraction
11001 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11002 
11003   match(Set dst (SubL src1 src2));
11004 
11005   ins_cost(INSN_COST);
11006   format %{ "sub  $dst, $src1, $src2" %}
11007 
11008   ins_encode %{
11009     __ sub(as_Register($dst$$reg),
11010            as_Register($src1$$reg),
11011            as_Register($src2$$reg));
11012   %}
11013 
11014   ins_pipe(ialu_reg_reg);
11015 %}
11016 
11017 // No constant pool entries requiredLong Immediate Subtraction.
11018 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
11019   match(Set dst (SubL src1 src2));
11020 
11021   ins_cost(INSN_COST);
11022   format %{ "sub$dst, $src1, $src2" %}
11023 
11024   // use opcode to indicate that this is a sub not an add
11025   opcode(0x1);
11026 
11027   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
11028 
11029   ins_pipe(ialu_reg_imm);
11030 %}
11031 
11032 // Integer Negation (special case for sub)
11033 
11034 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
11035   match(Set dst (SubI zero src));
11036 
11037   ins_cost(INSN_COST);
11038   format %{ "negw $dst, $src\t# int" %}
11039 
11040   ins_encode %{
11041     __ negw(as_Register($dst$$reg),
11042             as_Register($src$$reg));
11043   %}
11044 
11045   ins_pipe(ialu_reg);
11046 %}
11047 
11048 // Long Negation
11049 
11050 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
11051   match(Set dst (SubL zero src));
11052 
11053   ins_cost(INSN_COST);
11054   format %{ "neg $dst, $src\t# long" %}
11055 
11056   ins_encode %{
11057     __ neg(as_Register($dst$$reg),
11058            as_Register($src$$reg));
11059   %}
11060 
11061   ins_pipe(ialu_reg);
11062 %}
11063 
11064 // Integer Multiply
11065 
11066 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11067   match(Set dst (MulI src1 src2));
11068 
11069   ins_cost(INSN_COST * 3);
11070   format %{ "mulw  $dst, $src1, $src2" %}
11071 
11072   ins_encode %{
11073     __ mulw(as_Register($dst$$reg),
11074             as_Register($src1$$reg),
11075             as_Register($src2$$reg));
11076   %}
11077 
11078   ins_pipe(imul_reg_reg);
11079 %}
11080 
11081 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11082   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
11083 
11084   ins_cost(INSN_COST * 3);
11085   format %{ "smull  $dst, $src1, $src2" %}
11086 
11087   ins_encode %{
11088     __ smull(as_Register($dst$$reg),
11089              as_Register($src1$$reg),
11090              as_Register($src2$$reg));
11091   %}
11092 
11093   ins_pipe(imul_reg_reg);
11094 %}
11095 
11096 // Long Multiply
11097 
11098 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11099   match(Set dst (MulL src1 src2));
11100 
11101   ins_cost(INSN_COST * 5);
11102   format %{ "mul  $dst, $src1, $src2" %}
11103 
11104   ins_encode %{
11105     __ mul(as_Register($dst$$reg),
11106            as_Register($src1$$reg),
11107            as_Register($src2$$reg));
11108   %}
11109 
11110   ins_pipe(lmul_reg_reg);
11111 %}
11112 
11113 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11114 %{
11115   match(Set dst (MulHiL src1 src2));
11116 
11117   ins_cost(INSN_COST * 7);
11118   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
11119 
11120   ins_encode %{
11121     __ smulh(as_Register($dst$$reg),
11122              as_Register($src1$$reg),
11123              as_Register($src2$$reg));
11124   %}
11125 
11126   ins_pipe(lmul_reg_reg);
11127 %}
11128 
11129 // Combined Integer Multiply & Add/Sub
11130 
11131 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11132   match(Set dst (AddI src3 (MulI src1 src2)));
11133 
11134   ins_cost(INSN_COST * 3);
11135   format %{ "madd  $dst, $src1, $src2, $src3" %}
11136 
11137   ins_encode %{
11138     __ maddw(as_Register($dst$$reg),
11139              as_Register($src1$$reg),
11140              as_Register($src2$$reg),
11141              as_Register($src3$$reg));
11142   %}
11143 
11144   ins_pipe(imac_reg_reg);
11145 %}
11146 
11147 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11148   match(Set dst (SubI src3 (MulI src1 src2)));
11149 
11150   ins_cost(INSN_COST * 3);
11151   format %{ "msub  $dst, $src1, $src2, $src3" %}
11152 
11153   ins_encode %{
11154     __ msubw(as_Register($dst$$reg),
11155              as_Register($src1$$reg),
11156              as_Register($src2$$reg),
11157              as_Register($src3$$reg));
11158   %}
11159 
11160   ins_pipe(imac_reg_reg);
11161 %}
11162 
11163 // Combined Integer Multiply & Neg
11164 
11165 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11166   match(Set dst (MulI (SubI zero src1) src2));
11167 
11168   ins_cost(INSN_COST * 3);
11169   format %{ "mneg  $dst, $src1, $src2" %}
11170 
11171   ins_encode %{
11172     __ mnegw(as_Register($dst$$reg),
11173              as_Register($src1$$reg),
11174              as_Register($src2$$reg));
11175   %}
11176 
11177   ins_pipe(imac_reg_reg);
11178 %}
11179 
11180 // Combined Long Multiply & Add/Sub
11181 
11182 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11183   match(Set dst (AddL src3 (MulL src1 src2)));
11184 
11185   ins_cost(INSN_COST * 5);
11186   format %{ "madd  $dst, $src1, $src2, $src3" %}
11187 
11188   ins_encode %{
11189     __ madd(as_Register($dst$$reg),
11190             as_Register($src1$$reg),
11191             as_Register($src2$$reg),
11192             as_Register($src3$$reg));
11193   %}
11194 
11195   ins_pipe(lmac_reg_reg);
11196 %}
11197 
11198 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11199   match(Set dst (SubL src3 (MulL src1 src2)));
11200 
11201   ins_cost(INSN_COST * 5);
11202   format %{ "msub  $dst, $src1, $src2, $src3" %}
11203 
11204   ins_encode %{
11205     __ msub(as_Register($dst$$reg),
11206             as_Register($src1$$reg),
11207             as_Register($src2$$reg),
11208             as_Register($src3$$reg));
11209   %}
11210 
11211   ins_pipe(lmac_reg_reg);
11212 %}
11213 
11214 // Combined Long Multiply & Neg
11215 
11216 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11217   match(Set dst (MulL (SubL zero src1) src2));
11218 
11219   ins_cost(INSN_COST * 5);
11220   format %{ "mneg  $dst, $src1, $src2" %}
11221 
11222   ins_encode %{
11223     __ mneg(as_Register($dst$$reg),
11224             as_Register($src1$$reg),
11225             as_Register($src2$$reg));
11226   %}
11227 
11228   ins_pipe(lmac_reg_reg);
11229 %}
11230 
11231 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11232 
11233 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11234   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11235 
11236   ins_cost(INSN_COST * 3);
11237   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11238 
11239   ins_encode %{
11240     __ smaddl(as_Register($dst$$reg),
11241               as_Register($src1$$reg),
11242               as_Register($src2$$reg),
11243               as_Register($src3$$reg));
11244   %}
11245 
11246   ins_pipe(imac_reg_reg);
11247 %}
11248 
11249 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11250   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11251 
11252   ins_cost(INSN_COST * 3);
11253   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11254 
11255   ins_encode %{
11256     __ smsubl(as_Register($dst$$reg),
11257               as_Register($src1$$reg),
11258               as_Register($src2$$reg),
11259               as_Register($src3$$reg));
11260   %}
11261 
11262   ins_pipe(imac_reg_reg);
11263 %}
11264 
11265 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11266   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11267 
11268   ins_cost(INSN_COST * 3);
11269   format %{ "smnegl  $dst, $src1, $src2" %}
11270 
11271   ins_encode %{
11272     __ smnegl(as_Register($dst$$reg),
11273               as_Register($src1$$reg),
11274               as_Register($src2$$reg));
11275   %}
11276 
11277   ins_pipe(imac_reg_reg);
11278 %}
11279 
11280 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11281 
11282 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11283   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11284 
11285   ins_cost(INSN_COST * 5);
11286   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11287             "maddw $dst, $src3, $src4, rscratch1" %}
11288 
11289   ins_encode %{
11290     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11291     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11292 
11293   ins_pipe(imac_reg_reg);
11294 %}
11295 
11296 // Integer Divide
11297 
11298 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11299   match(Set dst (DivI src1 src2));
11300 
11301   ins_cost(INSN_COST * 19);
11302   format %{ "sdivw  $dst, $src1, $src2" %}
11303 
11304   ins_encode(aarch64_enc_divw(dst, src1, src2));
11305   ins_pipe(idiv_reg_reg);
11306 %}
11307 
11308 // Long Divide
11309 
11310 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11311   match(Set dst (DivL src1 src2));
11312 
11313   ins_cost(INSN_COST * 35);
11314   format %{ "sdiv   $dst, $src1, $src2" %}
11315 
11316   ins_encode(aarch64_enc_div(dst, src1, src2));
11317   ins_pipe(ldiv_reg_reg);
11318 %}
11319 
11320 // Integer Remainder
11321 
11322 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11323   match(Set dst (ModI src1 src2));
11324 
11325   ins_cost(INSN_COST * 22);
11326   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11327             "msubw($dst, rscratch1, $src2, $src1" %}
11328 
11329   ins_encode(aarch64_enc_modw(dst, src1, src2));
11330   ins_pipe(idiv_reg_reg);
11331 %}
11332 
11333 // Long Remainder
11334 
11335 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11336   match(Set dst (ModL src1 src2));
11337 
11338   ins_cost(INSN_COST * 38);
11339   format %{ "sdiv   rscratch1, $src1, $src2\n"
11340             "msub($dst, rscratch1, $src2, $src1" %}
11341 
11342   ins_encode(aarch64_enc_mod(dst, src1, src2));
11343   ins_pipe(ldiv_reg_reg);
11344 %}
11345 
11346 // Integer Shifts
11347 
11348 // Shift Left Register
11349 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11350   match(Set dst (LShiftI src1 src2));
11351 
11352   ins_cost(INSN_COST * 2);
11353   format %{ "lslvw  $dst, $src1, $src2" %}
11354 
11355   ins_encode %{
11356     __ lslvw(as_Register($dst$$reg),
11357              as_Register($src1$$reg),
11358              as_Register($src2$$reg));
11359   %}
11360 
11361   ins_pipe(ialu_reg_reg_vshift);
11362 %}
11363 
11364 // Shift Left Immediate
11365 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11366   match(Set dst (LShiftI src1 src2));
11367 
11368   ins_cost(INSN_COST);
11369   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11370 
11371   ins_encode %{
11372     __ lslw(as_Register($dst$$reg),
11373             as_Register($src1$$reg),
11374             $src2$$constant & 0x1f);
11375   %}
11376 
11377   ins_pipe(ialu_reg_shift);
11378 %}
11379 
11380 // Shift Right Logical Register
11381 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11382   match(Set dst (URShiftI src1 src2));
11383 
11384   ins_cost(INSN_COST * 2);
11385   format %{ "lsrvw  $dst, $src1, $src2" %}
11386 
11387   ins_encode %{
11388     __ lsrvw(as_Register($dst$$reg),
11389              as_Register($src1$$reg),
11390              as_Register($src2$$reg));
11391   %}
11392 
11393   ins_pipe(ialu_reg_reg_vshift);
11394 %}
11395 
11396 // Shift Right Logical Immediate
11397 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11398   match(Set dst (URShiftI src1 src2));
11399 
11400   ins_cost(INSN_COST);
11401   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11402 
11403   ins_encode %{
11404     __ lsrw(as_Register($dst$$reg),
11405             as_Register($src1$$reg),
11406             $src2$$constant & 0x1f);
11407   %}
11408 
11409   ins_pipe(ialu_reg_shift);
11410 %}
11411 
11412 // Shift Right Arithmetic Register
11413 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11414   match(Set dst (RShiftI src1 src2));
11415 
11416   ins_cost(INSN_COST * 2);
11417   format %{ "asrvw  $dst, $src1, $src2" %}
11418 
11419   ins_encode %{
11420     __ asrvw(as_Register($dst$$reg),
11421              as_Register($src1$$reg),
11422              as_Register($src2$$reg));
11423   %}
11424 
11425   ins_pipe(ialu_reg_reg_vshift);
11426 %}
11427 
11428 // Shift Right Arithmetic Immediate
11429 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11430   match(Set dst (RShiftI src1 src2));
11431 
11432   ins_cost(INSN_COST);
11433   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11434 
11435   ins_encode %{
11436     __ asrw(as_Register($dst$$reg),
11437             as_Register($src1$$reg),
11438             $src2$$constant & 0x1f);
11439   %}
11440 
11441   ins_pipe(ialu_reg_shift);
11442 %}
11443 
11444 // Combined Int Mask and Right Shift (using UBFM)
11445 // TODO
11446 
11447 // Long Shifts
11448 
11449 // Shift Left Register
11450 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11451   match(Set dst (LShiftL src1 src2));
11452 
11453   ins_cost(INSN_COST * 2);
11454   format %{ "lslv  $dst, $src1, $src2" %}
11455 
11456   ins_encode %{
11457     __ lslv(as_Register($dst$$reg),
11458             as_Register($src1$$reg),
11459             as_Register($src2$$reg));
11460   %}
11461 
11462   ins_pipe(ialu_reg_reg_vshift);
11463 %}
11464 
11465 // Shift Left Immediate
11466 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11467   match(Set dst (LShiftL src1 src2));
11468 
11469   ins_cost(INSN_COST);
11470   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11471 
11472   ins_encode %{
11473     __ lsl(as_Register($dst$$reg),
11474             as_Register($src1$$reg),
11475             $src2$$constant & 0x3f);
11476   %}
11477 
11478   ins_pipe(ialu_reg_shift);
11479 %}
11480 
11481 // Shift Right Logical Register
11482 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11483   match(Set dst (URShiftL src1 src2));
11484 
11485   ins_cost(INSN_COST * 2);
11486   format %{ "lsrv  $dst, $src1, $src2" %}
11487 
11488   ins_encode %{
11489     __ lsrv(as_Register($dst$$reg),
11490             as_Register($src1$$reg),
11491             as_Register($src2$$reg));
11492   %}
11493 
11494   ins_pipe(ialu_reg_reg_vshift);
11495 %}
11496 
11497 // Shift Right Logical Immediate
11498 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11499   match(Set dst (URShiftL src1 src2));
11500 
11501   ins_cost(INSN_COST);
11502   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11503 
11504   ins_encode %{
11505     __ lsr(as_Register($dst$$reg),
11506            as_Register($src1$$reg),
11507            $src2$$constant & 0x3f);
11508   %}
11509 
11510   ins_pipe(ialu_reg_shift);
11511 %}
11512 
11513 // A special-case pattern for card table stores.
11514 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11515   match(Set dst (URShiftL (CastP2X src1) src2));
11516 
11517   ins_cost(INSN_COST);
11518   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11519 
11520   ins_encode %{
11521     __ lsr(as_Register($dst$$reg),
11522            as_Register($src1$$reg),
11523            $src2$$constant & 0x3f);
11524   %}
11525 
11526   ins_pipe(ialu_reg_shift);
11527 %}
11528 
11529 // Shift Right Arithmetic Register
11530 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11531   match(Set dst (RShiftL src1 src2));
11532 
11533   ins_cost(INSN_COST * 2);
11534   format %{ "asrv  $dst, $src1, $src2" %}
11535 
11536   ins_encode %{
11537     __ asrv(as_Register($dst$$reg),
11538             as_Register($src1$$reg),
11539             as_Register($src2$$reg));
11540   %}
11541 
11542   ins_pipe(ialu_reg_reg_vshift);
11543 %}
11544 
11545 // Shift Right Arithmetic Immediate
11546 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11547   match(Set dst (RShiftL src1 src2));
11548 
11549   ins_cost(INSN_COST);
11550   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11551 
11552   ins_encode %{
11553     __ asr(as_Register($dst$$reg),
11554            as_Register($src1$$reg),
11555            $src2$$constant & 0x3f);
11556   %}
11557 
11558   ins_pipe(ialu_reg_shift);
11559 %}
11560 
11561 // BEGIN This section of the file is automatically generated. Do not edit --------------
11562 // This section is generated from aarch64_ad.m4
11563 
11564 
11565 // This pattern is automatically generated from aarch64_ad.m4
11566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11567 instruct regL_not_reg(iRegLNoSp dst,
11568                          iRegL src1, immL_M1 m1,
11569                          rFlagsReg cr) %{
11570   match(Set dst (XorL src1 m1));
11571   ins_cost(INSN_COST);
11572   format %{ "eon  $dst, $src1, zr" %}
11573 
11574   ins_encode %{
11575     __ eon(as_Register($dst$$reg),
11576               as_Register($src1$$reg),
11577               zr,
11578               Assembler::LSL, 0);
11579   %}
11580 
11581   ins_pipe(ialu_reg);
11582 %}
11583 
11584 // This pattern is automatically generated from aarch64_ad.m4
11585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11586 instruct regI_not_reg(iRegINoSp dst,
11587                          iRegIorL2I src1, immI_M1 m1,
11588                          rFlagsReg cr) %{
11589   match(Set dst (XorI src1 m1));
11590   ins_cost(INSN_COST);
11591   format %{ "eonw  $dst, $src1, zr" %}
11592 
11593   ins_encode %{
11594     __ eonw(as_Register($dst$$reg),
11595               as_Register($src1$$reg),
11596               zr,
11597               Assembler::LSL, 0);
11598   %}
11599 
11600   ins_pipe(ialu_reg);
11601 %}
11602 
11603 // This pattern is automatically generated from aarch64_ad.m4
11604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11605 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11606                               immI0 zero, iRegIorL2I src1, immI src2) %{
11607   match(Set dst (SubI zero (URShiftI src1 src2)));
11608 
11609   ins_cost(1.9 * INSN_COST);
11610   format %{ "negw  $dst, $src1, LSR $src2" %}
11611 
11612   ins_encode %{
11613     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11614             Assembler::LSR, $src2$$constant & 0x1f);
11615   %}
11616 
11617   ins_pipe(ialu_reg_shift);
11618 %}
11619 
11620 // This pattern is automatically generated from aarch64_ad.m4
11621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11622 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11623                               immI0 zero, iRegIorL2I src1, immI src2) %{
11624   match(Set dst (SubI zero (RShiftI src1 src2)));
11625 
11626   ins_cost(1.9 * INSN_COST);
11627   format %{ "negw  $dst, $src1, ASR $src2" %}
11628 
11629   ins_encode %{
11630     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11631             Assembler::ASR, $src2$$constant & 0x1f);
11632   %}
11633 
11634   ins_pipe(ialu_reg_shift);
11635 %}
11636 
11637 // This pattern is automatically generated from aarch64_ad.m4
11638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11639 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11640                               immI0 zero, iRegIorL2I src1, immI src2) %{
11641   match(Set dst (SubI zero (LShiftI src1 src2)));
11642 
11643   ins_cost(1.9 * INSN_COST);
11644   format %{ "negw  $dst, $src1, LSL $src2" %}
11645 
11646   ins_encode %{
11647     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11648             Assembler::LSL, $src2$$constant & 0x1f);
11649   %}
11650 
11651   ins_pipe(ialu_reg_shift);
11652 %}
11653 
11654 // This pattern is automatically generated from aarch64_ad.m4
11655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11656 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11657                               immL0 zero, iRegL src1, immI src2) %{
11658   match(Set dst (SubL zero (URShiftL src1 src2)));
11659 
11660   ins_cost(1.9 * INSN_COST);
11661   format %{ "neg  $dst, $src1, LSR $src2" %}
11662 
11663   ins_encode %{
11664     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11665             Assembler::LSR, $src2$$constant & 0x3f);
11666   %}
11667 
11668   ins_pipe(ialu_reg_shift);
11669 %}
11670 
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11674                               immL0 zero, iRegL src1, immI src2) %{
11675   match(Set dst (SubL zero (RShiftL src1 src2)));
11676 
11677   ins_cost(1.9 * INSN_COST);
11678   format %{ "neg  $dst, $src1, ASR $src2" %}
11679 
11680   ins_encode %{
11681     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11682             Assembler::ASR, $src2$$constant & 0x3f);
11683   %}
11684 
11685   ins_pipe(ialu_reg_shift);
11686 %}
11687 
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11691                               immL0 zero, iRegL src1, immI src2) %{
11692   match(Set dst (SubL zero (LShiftL src1 src2)));
11693 
11694   ins_cost(1.9 * INSN_COST);
11695   format %{ "neg  $dst, $src1, LSL $src2" %}
11696 
11697   ins_encode %{
11698     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11699             Assembler::LSL, $src2$$constant & 0x3f);
11700   %}
11701 
11702   ins_pipe(ialu_reg_shift);
11703 %}
11704 
11705 // This pattern is automatically generated from aarch64_ad.m4
11706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11707 instruct AndI_reg_not_reg(iRegINoSp dst,
11708                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11709   match(Set dst (AndI src1 (XorI src2 m1)));
11710   ins_cost(INSN_COST);
11711   format %{ "bicw  $dst, $src1, $src2" %}
11712 
11713   ins_encode %{
11714     __ bicw(as_Register($dst$$reg),
11715               as_Register($src1$$reg),
11716               as_Register($src2$$reg),
11717               Assembler::LSL, 0);
11718   %}
11719 
11720   ins_pipe(ialu_reg_reg);
11721 %}
11722 
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 instruct AndL_reg_not_reg(iRegLNoSp dst,
11726                          iRegL src1, iRegL src2, immL_M1 m1) %{
11727   match(Set dst (AndL src1 (XorL src2 m1)));
11728   ins_cost(INSN_COST);
11729   format %{ "bic  $dst, $src1, $src2" %}
11730 
11731   ins_encode %{
11732     __ bic(as_Register($dst$$reg),
11733               as_Register($src1$$reg),
11734               as_Register($src2$$reg),
11735               Assembler::LSL, 0);
11736   %}
11737 
11738   ins_pipe(ialu_reg_reg);
11739 %}
11740 
11741 // This pattern is automatically generated from aarch64_ad.m4
11742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11743 instruct OrI_reg_not_reg(iRegINoSp dst,
11744                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11745   match(Set dst (OrI src1 (XorI src2 m1)));
11746   ins_cost(INSN_COST);
11747   format %{ "ornw  $dst, $src1, $src2" %}
11748 
11749   ins_encode %{
11750     __ ornw(as_Register($dst$$reg),
11751               as_Register($src1$$reg),
11752               as_Register($src2$$reg),
11753               Assembler::LSL, 0);
11754   %}
11755 
11756   ins_pipe(ialu_reg_reg);
11757 %}
11758 
11759 // This pattern is automatically generated from aarch64_ad.m4
11760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11761 instruct OrL_reg_not_reg(iRegLNoSp dst,
11762                          iRegL src1, iRegL src2, immL_M1 m1) %{
11763   match(Set dst (OrL src1 (XorL src2 m1)));
11764   ins_cost(INSN_COST);
11765   format %{ "orn  $dst, $src1, $src2" %}
11766 
11767   ins_encode %{
11768     __ orn(as_Register($dst$$reg),
11769               as_Register($src1$$reg),
11770               as_Register($src2$$reg),
11771               Assembler::LSL, 0);
11772   %}
11773 
11774   ins_pipe(ialu_reg_reg);
11775 %}
11776 
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 instruct XorI_reg_not_reg(iRegINoSp dst,
11780                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11781   match(Set dst (XorI m1 (XorI src2 src1)));
11782   ins_cost(INSN_COST);
11783   format %{ "eonw  $dst, $src1, $src2" %}
11784 
11785   ins_encode %{
11786     __ eonw(as_Register($dst$$reg),
11787               as_Register($src1$$reg),
11788               as_Register($src2$$reg),
11789               Assembler::LSL, 0);
11790   %}
11791 
11792   ins_pipe(ialu_reg_reg);
11793 %}
11794 
11795 // This pattern is automatically generated from aarch64_ad.m4
11796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11797 instruct XorL_reg_not_reg(iRegLNoSp dst,
11798                          iRegL src1, iRegL src2, immL_M1 m1) %{
11799   match(Set dst (XorL m1 (XorL src2 src1)));
11800   ins_cost(INSN_COST);
11801   format %{ "eon  $dst, $src1, $src2" %}
11802 
11803   ins_encode %{
11804     __ eon(as_Register($dst$$reg),
11805               as_Register($src1$$reg),
11806               as_Register($src2$$reg),
11807               Assembler::LSL, 0);
11808   %}
11809 
11810   ins_pipe(ialu_reg_reg);
11811 %}
11812 
11813 // This pattern is automatically generated from aarch64_ad.m4
11814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11815 // val & (-1 ^ (val >>> shift)) ==> bicw
11816 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11817                          iRegIorL2I src1, iRegIorL2I src2,
11818                          immI src3, immI_M1 src4) %{
11819   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11820   ins_cost(1.9 * INSN_COST);
11821   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11822 
11823   ins_encode %{
11824     __ bicw(as_Register($dst$$reg),
11825               as_Register($src1$$reg),
11826               as_Register($src2$$reg),
11827               Assembler::LSR,
11828               $src3$$constant & 0x1f);
11829   %}
11830 
11831   ins_pipe(ialu_reg_reg_shift);
11832 %}
11833 
11834 // This pattern is automatically generated from aarch64_ad.m4
11835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11836 // val & (-1 ^ (val >>> shift)) ==> bic
11837 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11838                          iRegL src1, iRegL src2,
11839                          immI src3, immL_M1 src4) %{
11840   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11841   ins_cost(1.9 * INSN_COST);
11842   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11843 
11844   ins_encode %{
11845     __ bic(as_Register($dst$$reg),
11846               as_Register($src1$$reg),
11847               as_Register($src2$$reg),
11848               Assembler::LSR,
11849               $src3$$constant & 0x3f);
11850   %}
11851 
11852   ins_pipe(ialu_reg_reg_shift);
11853 %}
11854 
11855 // This pattern is automatically generated from aarch64_ad.m4
11856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11857 // val & (-1 ^ (val >> shift)) ==> bicw
11858 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11859                          iRegIorL2I src1, iRegIorL2I src2,
11860                          immI src3, immI_M1 src4) %{
11861   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11862   ins_cost(1.9 * INSN_COST);
11863   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11864 
11865   ins_encode %{
11866     __ bicw(as_Register($dst$$reg),
11867               as_Register($src1$$reg),
11868               as_Register($src2$$reg),
11869               Assembler::ASR,
11870               $src3$$constant & 0x1f);
11871   %}
11872 
11873   ins_pipe(ialu_reg_reg_shift);
11874 %}
11875 
11876 // This pattern is automatically generated from aarch64_ad.m4
11877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11878 // val & (-1 ^ (val >> shift)) ==> bic
11879 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11880                          iRegL src1, iRegL src2,
11881                          immI src3, immL_M1 src4) %{
11882   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11883   ins_cost(1.9 * INSN_COST);
11884   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11885 
11886   ins_encode %{
11887     __ bic(as_Register($dst$$reg),
11888               as_Register($src1$$reg),
11889               as_Register($src2$$reg),
11890               Assembler::ASR,
11891               $src3$$constant & 0x3f);
11892   %}
11893 
11894   ins_pipe(ialu_reg_reg_shift);
11895 %}
11896 
11897 // This pattern is automatically generated from aarch64_ad.m4
11898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11899 // val & (-1 ^ (val ror shift)) ==> bicw
11900 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11901                          iRegIorL2I src1, iRegIorL2I src2,
11902                          immI src3, immI_M1 src4) %{
11903   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11904   ins_cost(1.9 * INSN_COST);
11905   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11906 
11907   ins_encode %{
11908     __ bicw(as_Register($dst$$reg),
11909               as_Register($src1$$reg),
11910               as_Register($src2$$reg),
11911               Assembler::ROR,
11912               $src3$$constant & 0x1f);
11913   %}
11914 
11915   ins_pipe(ialu_reg_reg_shift);
11916 %}
11917 
11918 // This pattern is automatically generated from aarch64_ad.m4
11919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11920 // val & (-1 ^ (val ror shift)) ==> bic
11921 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11922                          iRegL src1, iRegL src2,
11923                          immI src3, immL_M1 src4) %{
11924   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11925   ins_cost(1.9 * INSN_COST);
11926   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11927 
11928   ins_encode %{
11929     __ bic(as_Register($dst$$reg),
11930               as_Register($src1$$reg),
11931               as_Register($src2$$reg),
11932               Assembler::ROR,
11933               $src3$$constant & 0x3f);
11934   %}
11935 
11936   ins_pipe(ialu_reg_reg_shift);
11937 %}
11938 
11939 // This pattern is automatically generated from aarch64_ad.m4
11940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11941 // val & (-1 ^ (val << shift)) ==> bicw
11942 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11943                          iRegIorL2I src1, iRegIorL2I src2,
11944                          immI src3, immI_M1 src4) %{
11945   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11946   ins_cost(1.9 * INSN_COST);
11947   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11948 
11949   ins_encode %{
11950     __ bicw(as_Register($dst$$reg),
11951               as_Register($src1$$reg),
11952               as_Register($src2$$reg),
11953               Assembler::LSL,
11954               $src3$$constant & 0x1f);
11955   %}
11956 
11957   ins_pipe(ialu_reg_reg_shift);
11958 %}
11959 
11960 // This pattern is automatically generated from aarch64_ad.m4
11961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11962 // val & (-1 ^ (val << shift)) ==> bic
11963 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11964                          iRegL src1, iRegL src2,
11965                          immI src3, immL_M1 src4) %{
11966   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11967   ins_cost(1.9 * INSN_COST);
11968   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11969 
11970   ins_encode %{
11971     __ bic(as_Register($dst$$reg),
11972               as_Register($src1$$reg),
11973               as_Register($src2$$reg),
11974               Assembler::LSL,
11975               $src3$$constant & 0x3f);
11976   %}
11977 
11978   ins_pipe(ialu_reg_reg_shift);
11979 %}
11980 
11981 // This pattern is automatically generated from aarch64_ad.m4
11982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11983 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11984 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11985                          iRegIorL2I src1, iRegIorL2I src2,
11986                          immI src3, immI_M1 src4) %{
11987   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11988   ins_cost(1.9 * INSN_COST);
11989   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11990 
11991   ins_encode %{
11992     __ eonw(as_Register($dst$$reg),
11993               as_Register($src1$$reg),
11994               as_Register($src2$$reg),
11995               Assembler::LSR,
11996               $src3$$constant & 0x1f);
11997   %}
11998 
11999   ins_pipe(ialu_reg_reg_shift);
12000 %}
12001 
12002 // This pattern is automatically generated from aarch64_ad.m4
12003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12004 // val ^ (-1 ^ (val >>> shift)) ==> eon
12005 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
12006                          iRegL src1, iRegL src2,
12007                          immI src3, immL_M1 src4) %{
12008   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
12009   ins_cost(1.9 * INSN_COST);
12010   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
12011 
12012   ins_encode %{
12013     __ eon(as_Register($dst$$reg),
12014               as_Register($src1$$reg),
12015               as_Register($src2$$reg),
12016               Assembler::LSR,
12017               $src3$$constant & 0x3f);
12018   %}
12019 
12020   ins_pipe(ialu_reg_reg_shift);
12021 %}
12022 
12023 // This pattern is automatically generated from aarch64_ad.m4
12024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12025 // val ^ (-1 ^ (val >> shift)) ==> eonw
12026 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
12027                          iRegIorL2I src1, iRegIorL2I src2,
12028                          immI src3, immI_M1 src4) %{
12029   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
12030   ins_cost(1.9 * INSN_COST);
12031   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
12032 
12033   ins_encode %{
12034     __ eonw(as_Register($dst$$reg),
12035               as_Register($src1$$reg),
12036               as_Register($src2$$reg),
12037               Assembler::ASR,
12038               $src3$$constant & 0x1f);
12039   %}
12040 
12041   ins_pipe(ialu_reg_reg_shift);
12042 %}
12043 
12044 // This pattern is automatically generated from aarch64_ad.m4
12045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12046 // val ^ (-1 ^ (val >> shift)) ==> eon
12047 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
12048                          iRegL src1, iRegL src2,
12049                          immI src3, immL_M1 src4) %{
12050   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
12051   ins_cost(1.9 * INSN_COST);
12052   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
12053 
12054   ins_encode %{
12055     __ eon(as_Register($dst$$reg),
12056               as_Register($src1$$reg),
12057               as_Register($src2$$reg),
12058               Assembler::ASR,
12059               $src3$$constant & 0x3f);
12060   %}
12061 
12062   ins_pipe(ialu_reg_reg_shift);
12063 %}
12064 
12065 // This pattern is automatically generated from aarch64_ad.m4
12066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12067 // val ^ (-1 ^ (val ror shift)) ==> eonw
12068 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
12069                          iRegIorL2I src1, iRegIorL2I src2,
12070                          immI src3, immI_M1 src4) %{
12071   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
12072   ins_cost(1.9 * INSN_COST);
12073   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
12074 
12075   ins_encode %{
12076     __ eonw(as_Register($dst$$reg),
12077               as_Register($src1$$reg),
12078               as_Register($src2$$reg),
12079               Assembler::ROR,
12080               $src3$$constant & 0x1f);
12081   %}
12082 
12083   ins_pipe(ialu_reg_reg_shift);
12084 %}
12085 
12086 // This pattern is automatically generated from aarch64_ad.m4
12087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12088 // val ^ (-1 ^ (val ror shift)) ==> eon
12089 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
12090                          iRegL src1, iRegL src2,
12091                          immI src3, immL_M1 src4) %{
12092   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
12093   ins_cost(1.9 * INSN_COST);
12094   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
12095 
12096   ins_encode %{
12097     __ eon(as_Register($dst$$reg),
12098               as_Register($src1$$reg),
12099               as_Register($src2$$reg),
12100               Assembler::ROR,
12101               $src3$$constant & 0x3f);
12102   %}
12103 
12104   ins_pipe(ialu_reg_reg_shift);
12105 %}
12106 
12107 // This pattern is automatically generated from aarch64_ad.m4
12108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12109 // val ^ (-1 ^ (val << shift)) ==> eonw
12110 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
12111                          iRegIorL2I src1, iRegIorL2I src2,
12112                          immI src3, immI_M1 src4) %{
12113   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
12114   ins_cost(1.9 * INSN_COST);
12115   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
12116 
12117   ins_encode %{
12118     __ eonw(as_Register($dst$$reg),
12119               as_Register($src1$$reg),
12120               as_Register($src2$$reg),
12121               Assembler::LSL,
12122               $src3$$constant & 0x1f);
12123   %}
12124 
12125   ins_pipe(ialu_reg_reg_shift);
12126 %}
12127 
12128 // This pattern is automatically generated from aarch64_ad.m4
12129 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12130 // val ^ (-1 ^ (val << shift)) ==> eon
12131 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
12132                          iRegL src1, iRegL src2,
12133                          immI src3, immL_M1 src4) %{
12134   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12135   ins_cost(1.9 * INSN_COST);
12136   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
12137 
12138   ins_encode %{
12139     __ eon(as_Register($dst$$reg),
12140               as_Register($src1$$reg),
12141               as_Register($src2$$reg),
12142               Assembler::LSL,
12143               $src3$$constant & 0x3f);
12144   %}
12145 
12146   ins_pipe(ialu_reg_reg_shift);
12147 %}
12148 
12149 // This pattern is automatically generated from aarch64_ad.m4
12150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12151 // val | (-1 ^ (val >>> shift)) ==> ornw
12152 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12153                          iRegIorL2I src1, iRegIorL2I src2,
12154                          immI src3, immI_M1 src4) %{
12155   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12156   ins_cost(1.9 * INSN_COST);
12157   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
12158 
12159   ins_encode %{
12160     __ ornw(as_Register($dst$$reg),
12161               as_Register($src1$$reg),
12162               as_Register($src2$$reg),
12163               Assembler::LSR,
12164               $src3$$constant & 0x1f);
12165   %}
12166 
12167   ins_pipe(ialu_reg_reg_shift);
12168 %}
12169 
12170 // This pattern is automatically generated from aarch64_ad.m4
12171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12172 // val | (-1 ^ (val >>> shift)) ==> orn
12173 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12174                          iRegL src1, iRegL src2,
12175                          immI src3, immL_M1 src4) %{
12176   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12177   ins_cost(1.9 * INSN_COST);
12178   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
12179 
12180   ins_encode %{
12181     __ orn(as_Register($dst$$reg),
12182               as_Register($src1$$reg),
12183               as_Register($src2$$reg),
12184               Assembler::LSR,
12185               $src3$$constant & 0x3f);
12186   %}
12187 
12188   ins_pipe(ialu_reg_reg_shift);
12189 %}
12190 
12191 // This pattern is automatically generated from aarch64_ad.m4
12192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12193 // val | (-1 ^ (val >> shift)) ==> ornw
12194 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12195                          iRegIorL2I src1, iRegIorL2I src2,
12196                          immI src3, immI_M1 src4) %{
12197   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12198   ins_cost(1.9 * INSN_COST);
12199   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12200 
12201   ins_encode %{
12202     __ ornw(as_Register($dst$$reg),
12203               as_Register($src1$$reg),
12204               as_Register($src2$$reg),
12205               Assembler::ASR,
12206               $src3$$constant & 0x1f);
12207   %}
12208 
12209   ins_pipe(ialu_reg_reg_shift);
12210 %}
12211 
12212 // This pattern is automatically generated from aarch64_ad.m4
12213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12214 // val | (-1 ^ (val >> shift)) ==> orn
12215 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12216                          iRegL src1, iRegL src2,
12217                          immI src3, immL_M1 src4) %{
12218   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12219   ins_cost(1.9 * INSN_COST);
12220   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12221 
12222   ins_encode %{
12223     __ orn(as_Register($dst$$reg),
12224               as_Register($src1$$reg),
12225               as_Register($src2$$reg),
12226               Assembler::ASR,
12227               $src3$$constant & 0x3f);
12228   %}
12229 
12230   ins_pipe(ialu_reg_reg_shift);
12231 %}
12232 
12233 // This pattern is automatically generated from aarch64_ad.m4
12234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12235 // val | (-1 ^ (val ror shift)) ==> ornw
12236 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12237                          iRegIorL2I src1, iRegIorL2I src2,
12238                          immI src3, immI_M1 src4) %{
12239   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12240   ins_cost(1.9 * INSN_COST);
12241   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12242 
12243   ins_encode %{
12244     __ ornw(as_Register($dst$$reg),
12245               as_Register($src1$$reg),
12246               as_Register($src2$$reg),
12247               Assembler::ROR,
12248               $src3$$constant & 0x1f);
12249   %}
12250 
12251   ins_pipe(ialu_reg_reg_shift);
12252 %}
12253 
12254 // This pattern is automatically generated from aarch64_ad.m4
12255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12256 // val | (-1 ^ (val ror shift)) ==> orn
12257 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12258                          iRegL src1, iRegL src2,
12259                          immI src3, immL_M1 src4) %{
12260   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12261   ins_cost(1.9 * INSN_COST);
12262   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12263 
12264   ins_encode %{
12265     __ orn(as_Register($dst$$reg),
12266               as_Register($src1$$reg),
12267               as_Register($src2$$reg),
12268               Assembler::ROR,
12269               $src3$$constant & 0x3f);
12270   %}
12271 
12272   ins_pipe(ialu_reg_reg_shift);
12273 %}
12274 
12275 // This pattern is automatically generated from aarch64_ad.m4
12276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12277 // val | (-1 ^ (val << shift)) ==> ornw
12278 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12279                          iRegIorL2I src1, iRegIorL2I src2,
12280                          immI src3, immI_M1 src4) %{
12281   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12282   ins_cost(1.9 * INSN_COST);
12283   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12284 
12285   ins_encode %{
12286     __ ornw(as_Register($dst$$reg),
12287               as_Register($src1$$reg),
12288               as_Register($src2$$reg),
12289               Assembler::LSL,
12290               $src3$$constant & 0x1f);
12291   %}
12292 
12293   ins_pipe(ialu_reg_reg_shift);
12294 %}
12295 
12296 // This pattern is automatically generated from aarch64_ad.m4
12297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12298 // val | (-1 ^ (val << shift)) ==> orn
12299 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12300                          iRegL src1, iRegL src2,
12301                          immI src3, immL_M1 src4) %{
12302   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12303   ins_cost(1.9 * INSN_COST);
12304   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12305 
12306   ins_encode %{
12307     __ orn(as_Register($dst$$reg),
12308               as_Register($src1$$reg),
12309               as_Register($src2$$reg),
12310               Assembler::LSL,
12311               $src3$$constant & 0x3f);
12312   %}
12313 
12314   ins_pipe(ialu_reg_reg_shift);
12315 %}
12316 
12317 // This pattern is automatically generated from aarch64_ad.m4
12318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12319 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12320                          iRegIorL2I src1, iRegIorL2I src2,
12321                          immI src3) %{
12322   match(Set dst (AndI src1 (URShiftI src2 src3)));
12323 
12324   ins_cost(1.9 * INSN_COST);
12325   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12326 
12327   ins_encode %{
12328     __ andw(as_Register($dst$$reg),
12329               as_Register($src1$$reg),
12330               as_Register($src2$$reg),
12331               Assembler::LSR,
12332               $src3$$constant & 0x1f);
12333   %}
12334 
12335   ins_pipe(ialu_reg_reg_shift);
12336 %}
12337 
12338 // This pattern is automatically generated from aarch64_ad.m4
12339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12340 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12341                          iRegL src1, iRegL src2,
12342                          immI src3) %{
12343   match(Set dst (AndL src1 (URShiftL src2 src3)));
12344 
12345   ins_cost(1.9 * INSN_COST);
12346   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12347 
12348   ins_encode %{
12349     __ andr(as_Register($dst$$reg),
12350               as_Register($src1$$reg),
12351               as_Register($src2$$reg),
12352               Assembler::LSR,
12353               $src3$$constant & 0x3f);
12354   %}
12355 
12356   ins_pipe(ialu_reg_reg_shift);
12357 %}
12358 
12359 // This pattern is automatically generated from aarch64_ad.m4
12360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12361 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12362                          iRegIorL2I src1, iRegIorL2I src2,
12363                          immI src3) %{
12364   match(Set dst (AndI src1 (RShiftI src2 src3)));
12365 
12366   ins_cost(1.9 * INSN_COST);
12367   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12368 
12369   ins_encode %{
12370     __ andw(as_Register($dst$$reg),
12371               as_Register($src1$$reg),
12372               as_Register($src2$$reg),
12373               Assembler::ASR,
12374               $src3$$constant & 0x1f);
12375   %}
12376 
12377   ins_pipe(ialu_reg_reg_shift);
12378 %}
12379 
12380 // This pattern is automatically generated from aarch64_ad.m4
12381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12382 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12383                          iRegL src1, iRegL src2,
12384                          immI src3) %{
12385   match(Set dst (AndL src1 (RShiftL src2 src3)));
12386 
12387   ins_cost(1.9 * INSN_COST);
12388   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12389 
12390   ins_encode %{
12391     __ andr(as_Register($dst$$reg),
12392               as_Register($src1$$reg),
12393               as_Register($src2$$reg),
12394               Assembler::ASR,
12395               $src3$$constant & 0x3f);
12396   %}
12397 
12398   ins_pipe(ialu_reg_reg_shift);
12399 %}
12400 
12401 // This pattern is automatically generated from aarch64_ad.m4
12402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12403 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12404                          iRegIorL2I src1, iRegIorL2I src2,
12405                          immI src3) %{
12406   match(Set dst (AndI src1 (LShiftI src2 src3)));
12407 
12408   ins_cost(1.9 * INSN_COST);
12409   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12410 
12411   ins_encode %{
12412     __ andw(as_Register($dst$$reg),
12413               as_Register($src1$$reg),
12414               as_Register($src2$$reg),
12415               Assembler::LSL,
12416               $src3$$constant & 0x1f);
12417   %}
12418 
12419   ins_pipe(ialu_reg_reg_shift);
12420 %}
12421 
12422 // This pattern is automatically generated from aarch64_ad.m4
12423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12424 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12425                          iRegL src1, iRegL src2,
12426                          immI src3) %{
12427   match(Set dst (AndL src1 (LShiftL src2 src3)));
12428 
12429   ins_cost(1.9 * INSN_COST);
12430   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12431 
12432   ins_encode %{
12433     __ andr(as_Register($dst$$reg),
12434               as_Register($src1$$reg),
12435               as_Register($src2$$reg),
12436               Assembler::LSL,
12437               $src3$$constant & 0x3f);
12438   %}
12439 
12440   ins_pipe(ialu_reg_reg_shift);
12441 %}
12442 
12443 // This pattern is automatically generated from aarch64_ad.m4
12444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12445 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12446                          iRegIorL2I src1, iRegIorL2I src2,
12447                          immI src3) %{
12448   match(Set dst (AndI src1 (RotateRight src2 src3)));
12449 
12450   ins_cost(1.9 * INSN_COST);
12451   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12452 
12453   ins_encode %{
12454     __ andw(as_Register($dst$$reg),
12455               as_Register($src1$$reg),
12456               as_Register($src2$$reg),
12457               Assembler::ROR,
12458               $src3$$constant & 0x1f);
12459   %}
12460 
12461   ins_pipe(ialu_reg_reg_shift);
12462 %}
12463 
12464 // This pattern is automatically generated from aarch64_ad.m4
12465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12466 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12467                          iRegL src1, iRegL src2,
12468                          immI src3) %{
12469   match(Set dst (AndL src1 (RotateRight src2 src3)));
12470 
12471   ins_cost(1.9 * INSN_COST);
12472   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12473 
12474   ins_encode %{
12475     __ andr(as_Register($dst$$reg),
12476               as_Register($src1$$reg),
12477               as_Register($src2$$reg),
12478               Assembler::ROR,
12479               $src3$$constant & 0x3f);
12480   %}
12481 
12482   ins_pipe(ialu_reg_reg_shift);
12483 %}
12484 
12485 // This pattern is automatically generated from aarch64_ad.m4
12486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12487 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12488                          iRegIorL2I src1, iRegIorL2I src2,
12489                          immI src3) %{
12490   match(Set dst (XorI src1 (URShiftI src2 src3)));
12491 
12492   ins_cost(1.9 * INSN_COST);
12493   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12494 
12495   ins_encode %{
12496     __ eorw(as_Register($dst$$reg),
12497               as_Register($src1$$reg),
12498               as_Register($src2$$reg),
12499               Assembler::LSR,
12500               $src3$$constant & 0x1f);
12501   %}
12502 
12503   ins_pipe(ialu_reg_reg_shift);
12504 %}
12505 
12506 // This pattern is automatically generated from aarch64_ad.m4
12507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12508 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12509                          iRegL src1, iRegL src2,
12510                          immI src3) %{
12511   match(Set dst (XorL src1 (URShiftL src2 src3)));
12512 
12513   ins_cost(1.9 * INSN_COST);
12514   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12515 
12516   ins_encode %{
12517     __ eor(as_Register($dst$$reg),
12518               as_Register($src1$$reg),
12519               as_Register($src2$$reg),
12520               Assembler::LSR,
12521               $src3$$constant & 0x3f);
12522   %}
12523 
12524   ins_pipe(ialu_reg_reg_shift);
12525 %}
12526 
12527 // This pattern is automatically generated from aarch64_ad.m4
12528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12529 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12530                          iRegIorL2I src1, iRegIorL2I src2,
12531                          immI src3) %{
12532   match(Set dst (XorI src1 (RShiftI src2 src3)));
12533 
12534   ins_cost(1.9 * INSN_COST);
12535   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12536 
12537   ins_encode %{
12538     __ eorw(as_Register($dst$$reg),
12539               as_Register($src1$$reg),
12540               as_Register($src2$$reg),
12541               Assembler::ASR,
12542               $src3$$constant & 0x1f);
12543   %}
12544 
12545   ins_pipe(ialu_reg_reg_shift);
12546 %}
12547 
12548 // This pattern is automatically generated from aarch64_ad.m4
12549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12550 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12551                          iRegL src1, iRegL src2,
12552                          immI src3) %{
12553   match(Set dst (XorL src1 (RShiftL src2 src3)));
12554 
12555   ins_cost(1.9 * INSN_COST);
12556   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12557 
12558   ins_encode %{
12559     __ eor(as_Register($dst$$reg),
12560               as_Register($src1$$reg),
12561               as_Register($src2$$reg),
12562               Assembler::ASR,
12563               $src3$$constant & 0x3f);
12564   %}
12565 
12566   ins_pipe(ialu_reg_reg_shift);
12567 %}
12568 
12569 // This pattern is automatically generated from aarch64_ad.m4
12570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12571 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12572                          iRegIorL2I src1, iRegIorL2I src2,
12573                          immI src3) %{
12574   match(Set dst (XorI src1 (LShiftI src2 src3)));
12575 
12576   ins_cost(1.9 * INSN_COST);
12577   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12578 
12579   ins_encode %{
12580     __ eorw(as_Register($dst$$reg),
12581               as_Register($src1$$reg),
12582               as_Register($src2$$reg),
12583               Assembler::LSL,
12584               $src3$$constant & 0x1f);
12585   %}
12586 
12587   ins_pipe(ialu_reg_reg_shift);
12588 %}
12589 
12590 // This pattern is automatically generated from aarch64_ad.m4
12591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12592 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12593                          iRegL src1, iRegL src2,
12594                          immI src3) %{
12595   match(Set dst (XorL src1 (LShiftL src2 src3)));
12596 
12597   ins_cost(1.9 * INSN_COST);
12598   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12599 
12600   ins_encode %{
12601     __ eor(as_Register($dst$$reg),
12602               as_Register($src1$$reg),
12603               as_Register($src2$$reg),
12604               Assembler::LSL,
12605               $src3$$constant & 0x3f);
12606   %}
12607 
12608   ins_pipe(ialu_reg_reg_shift);
12609 %}
12610 
12611 // This pattern is automatically generated from aarch64_ad.m4
12612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12613 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12614                          iRegIorL2I src1, iRegIorL2I src2,
12615                          immI src3) %{
12616   match(Set dst (XorI src1 (RotateRight src2 src3)));
12617 
12618   ins_cost(1.9 * INSN_COST);
12619   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12620 
12621   ins_encode %{
12622     __ eorw(as_Register($dst$$reg),
12623               as_Register($src1$$reg),
12624               as_Register($src2$$reg),
12625               Assembler::ROR,
12626               $src3$$constant & 0x1f);
12627   %}
12628 
12629   ins_pipe(ialu_reg_reg_shift);
12630 %}
12631 
12632 // This pattern is automatically generated from aarch64_ad.m4
12633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12634 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12635                          iRegL src1, iRegL src2,
12636                          immI src3) %{
12637   match(Set dst (XorL src1 (RotateRight src2 src3)));
12638 
12639   ins_cost(1.9 * INSN_COST);
12640   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12641 
12642   ins_encode %{
12643     __ eor(as_Register($dst$$reg),
12644               as_Register($src1$$reg),
12645               as_Register($src2$$reg),
12646               Assembler::ROR,
12647               $src3$$constant & 0x3f);
12648   %}
12649 
12650   ins_pipe(ialu_reg_reg_shift);
12651 %}
12652 
12653 // This pattern is automatically generated from aarch64_ad.m4
12654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12655 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12656                          iRegIorL2I src1, iRegIorL2I src2,
12657                          immI src3) %{
12658   match(Set dst (OrI src1 (URShiftI src2 src3)));
12659 
12660   ins_cost(1.9 * INSN_COST);
12661   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12662 
12663   ins_encode %{
12664     __ orrw(as_Register($dst$$reg),
12665               as_Register($src1$$reg),
12666               as_Register($src2$$reg),
12667               Assembler::LSR,
12668               $src3$$constant & 0x1f);
12669   %}
12670 
12671   ins_pipe(ialu_reg_reg_shift);
12672 %}
12673 
12674 // This pattern is automatically generated from aarch64_ad.m4
12675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12676 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12677                          iRegL src1, iRegL src2,
12678                          immI src3) %{
12679   match(Set dst (OrL src1 (URShiftL src2 src3)));
12680 
12681   ins_cost(1.9 * INSN_COST);
12682   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12683 
12684   ins_encode %{
12685     __ orr(as_Register($dst$$reg),
12686               as_Register($src1$$reg),
12687               as_Register($src2$$reg),
12688               Assembler::LSR,
12689               $src3$$constant & 0x3f);
12690   %}
12691 
12692   ins_pipe(ialu_reg_reg_shift);
12693 %}
12694 
12695 // This pattern is automatically generated from aarch64_ad.m4
12696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12697 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12698                          iRegIorL2I src1, iRegIorL2I src2,
12699                          immI src3) %{
12700   match(Set dst (OrI src1 (RShiftI src2 src3)));
12701 
12702   ins_cost(1.9 * INSN_COST);
12703   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12704 
12705   ins_encode %{
12706     __ orrw(as_Register($dst$$reg),
12707               as_Register($src1$$reg),
12708               as_Register($src2$$reg),
12709               Assembler::ASR,
12710               $src3$$constant & 0x1f);
12711   %}
12712 
12713   ins_pipe(ialu_reg_reg_shift);
12714 %}
12715 
12716 // This pattern is automatically generated from aarch64_ad.m4
12717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12718 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12719                          iRegL src1, iRegL src2,
12720                          immI src3) %{
12721   match(Set dst (OrL src1 (RShiftL src2 src3)));
12722 
12723   ins_cost(1.9 * INSN_COST);
12724   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12725 
12726   ins_encode %{
12727     __ orr(as_Register($dst$$reg),
12728               as_Register($src1$$reg),
12729               as_Register($src2$$reg),
12730               Assembler::ASR,
12731               $src3$$constant & 0x3f);
12732   %}
12733 
12734   ins_pipe(ialu_reg_reg_shift);
12735 %}
12736 
12737 // This pattern is automatically generated from aarch64_ad.m4
12738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12739 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12740                          iRegIorL2I src1, iRegIorL2I src2,
12741                          immI src3) %{
12742   match(Set dst (OrI src1 (LShiftI src2 src3)));
12743 
12744   ins_cost(1.9 * INSN_COST);
12745   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12746 
12747   ins_encode %{
12748     __ orrw(as_Register($dst$$reg),
12749               as_Register($src1$$reg),
12750               as_Register($src2$$reg),
12751               Assembler::LSL,
12752               $src3$$constant & 0x1f);
12753   %}
12754 
12755   ins_pipe(ialu_reg_reg_shift);
12756 %}
12757 
12758 // This pattern is automatically generated from aarch64_ad.m4
12759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12760 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12761                          iRegL src1, iRegL src2,
12762                          immI src3) %{
12763   match(Set dst (OrL src1 (LShiftL src2 src3)));
12764 
12765   ins_cost(1.9 * INSN_COST);
12766   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12767 
12768   ins_encode %{
12769     __ orr(as_Register($dst$$reg),
12770               as_Register($src1$$reg),
12771               as_Register($src2$$reg),
12772               Assembler::LSL,
12773               $src3$$constant & 0x3f);
12774   %}
12775 
12776   ins_pipe(ialu_reg_reg_shift);
12777 %}
12778 
12779 // This pattern is automatically generated from aarch64_ad.m4
12780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12781 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12782                          iRegIorL2I src1, iRegIorL2I src2,
12783                          immI src3) %{
12784   match(Set dst (OrI src1 (RotateRight src2 src3)));
12785 
12786   ins_cost(1.9 * INSN_COST);
12787   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12788 
12789   ins_encode %{
12790     __ orrw(as_Register($dst$$reg),
12791               as_Register($src1$$reg),
12792               as_Register($src2$$reg),
12793               Assembler::ROR,
12794               $src3$$constant & 0x1f);
12795   %}
12796 
12797   ins_pipe(ialu_reg_reg_shift);
12798 %}
12799 
12800 // This pattern is automatically generated from aarch64_ad.m4
12801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12802 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12803                          iRegL src1, iRegL src2,
12804                          immI src3) %{
12805   match(Set dst (OrL src1 (RotateRight src2 src3)));
12806 
12807   ins_cost(1.9 * INSN_COST);
12808   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12809 
12810   ins_encode %{
12811     __ orr(as_Register($dst$$reg),
12812               as_Register($src1$$reg),
12813               as_Register($src2$$reg),
12814               Assembler::ROR,
12815               $src3$$constant & 0x3f);
12816   %}
12817 
12818   ins_pipe(ialu_reg_reg_shift);
12819 %}
12820 
12821 // This pattern is automatically generated from aarch64_ad.m4
12822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12823 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12824                          iRegIorL2I src1, iRegIorL2I src2,
12825                          immI src3) %{
12826   match(Set dst (AddI src1 (URShiftI src2 src3)));
12827 
12828   ins_cost(1.9 * INSN_COST);
12829   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12830 
12831   ins_encode %{
12832     __ addw(as_Register($dst$$reg),
12833               as_Register($src1$$reg),
12834               as_Register($src2$$reg),
12835               Assembler::LSR,
12836               $src3$$constant & 0x1f);
12837   %}
12838 
12839   ins_pipe(ialu_reg_reg_shift);
12840 %}
12841 
12842 // This pattern is automatically generated from aarch64_ad.m4
12843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12844 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12845                          iRegL src1, iRegL src2,
12846                          immI src3) %{
12847   match(Set dst (AddL src1 (URShiftL src2 src3)));
12848 
12849   ins_cost(1.9 * INSN_COST);
12850   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12851 
12852   ins_encode %{
12853     __ add(as_Register($dst$$reg),
12854               as_Register($src1$$reg),
12855               as_Register($src2$$reg),
12856               Assembler::LSR,
12857               $src3$$constant & 0x3f);
12858   %}
12859 
12860   ins_pipe(ialu_reg_reg_shift);
12861 %}
12862 
12863 // This pattern is automatically generated from aarch64_ad.m4
12864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12865 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12866                          iRegIorL2I src1, iRegIorL2I src2,
12867                          immI src3) %{
12868   match(Set dst (AddI src1 (RShiftI src2 src3)));
12869 
12870   ins_cost(1.9 * INSN_COST);
12871   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12872 
12873   ins_encode %{
12874     __ addw(as_Register($dst$$reg),
12875               as_Register($src1$$reg),
12876               as_Register($src2$$reg),
12877               Assembler::ASR,
12878               $src3$$constant & 0x1f);
12879   %}
12880 
12881   ins_pipe(ialu_reg_reg_shift);
12882 %}
12883 
12884 // This pattern is automatically generated from aarch64_ad.m4
12885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12886 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12887                          iRegL src1, iRegL src2,
12888                          immI src3) %{
12889   match(Set dst (AddL src1 (RShiftL src2 src3)));
12890 
12891   ins_cost(1.9 * INSN_COST);
12892   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12893 
12894   ins_encode %{
12895     __ add(as_Register($dst$$reg),
12896               as_Register($src1$$reg),
12897               as_Register($src2$$reg),
12898               Assembler::ASR,
12899               $src3$$constant & 0x3f);
12900   %}
12901 
12902   ins_pipe(ialu_reg_reg_shift);
12903 %}
12904 
12905 // This pattern is automatically generated from aarch64_ad.m4
12906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12907 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12908                          iRegIorL2I src1, iRegIorL2I src2,
12909                          immI src3) %{
12910   match(Set dst (AddI src1 (LShiftI src2 src3)));
12911 
12912   ins_cost(1.9 * INSN_COST);
12913   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12914 
12915   ins_encode %{
12916     __ addw(as_Register($dst$$reg),
12917               as_Register($src1$$reg),
12918               as_Register($src2$$reg),
12919               Assembler::LSL,
12920               $src3$$constant & 0x1f);
12921   %}
12922 
12923   ins_pipe(ialu_reg_reg_shift);
12924 %}
12925 
12926 // This pattern is automatically generated from aarch64_ad.m4
12927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12928 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12929                          iRegL src1, iRegL src2,
12930                          immI src3) %{
12931   match(Set dst (AddL src1 (LShiftL src2 src3)));
12932 
12933   ins_cost(1.9 * INSN_COST);
12934   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12935 
12936   ins_encode %{
12937     __ add(as_Register($dst$$reg),
12938               as_Register($src1$$reg),
12939               as_Register($src2$$reg),
12940               Assembler::LSL,
12941               $src3$$constant & 0x3f);
12942   %}
12943 
12944   ins_pipe(ialu_reg_reg_shift);
12945 %}
12946 
12947 // This pattern is automatically generated from aarch64_ad.m4
12948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12949 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12950                          iRegIorL2I src1, iRegIorL2I src2,
12951                          immI src3) %{
12952   match(Set dst (SubI src1 (URShiftI src2 src3)));
12953 
12954   ins_cost(1.9 * INSN_COST);
12955   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12956 
12957   ins_encode %{
12958     __ subw(as_Register($dst$$reg),
12959               as_Register($src1$$reg),
12960               as_Register($src2$$reg),
12961               Assembler::LSR,
12962               $src3$$constant & 0x1f);
12963   %}
12964 
12965   ins_pipe(ialu_reg_reg_shift);
12966 %}
12967 
12968 // This pattern is automatically generated from aarch64_ad.m4
12969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12970 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12971                          iRegL src1, iRegL src2,
12972                          immI src3) %{
12973   match(Set dst (SubL src1 (URShiftL src2 src3)));
12974 
12975   ins_cost(1.9 * INSN_COST);
12976   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12977 
12978   ins_encode %{
12979     __ sub(as_Register($dst$$reg),
12980               as_Register($src1$$reg),
12981               as_Register($src2$$reg),
12982               Assembler::LSR,
12983               $src3$$constant & 0x3f);
12984   %}
12985 
12986   ins_pipe(ialu_reg_reg_shift);
12987 %}
12988 
12989 // This pattern is automatically generated from aarch64_ad.m4
12990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12991 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12992                          iRegIorL2I src1, iRegIorL2I src2,
12993                          immI src3) %{
12994   match(Set dst (SubI src1 (RShiftI src2 src3)));
12995 
12996   ins_cost(1.9 * INSN_COST);
12997   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12998 
12999   ins_encode %{
13000     __ subw(as_Register($dst$$reg),
13001               as_Register($src1$$reg),
13002               as_Register($src2$$reg),
13003               Assembler::ASR,
13004               $src3$$constant & 0x1f);
13005   %}
13006 
13007   ins_pipe(ialu_reg_reg_shift);
13008 %}
13009 
13010 // This pattern is automatically generated from aarch64_ad.m4
13011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13012 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
13013                          iRegL src1, iRegL src2,
13014                          immI src3) %{
13015   match(Set dst (SubL src1 (RShiftL src2 src3)));
13016 
13017   ins_cost(1.9 * INSN_COST);
13018   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
13019 
13020   ins_encode %{
13021     __ sub(as_Register($dst$$reg),
13022               as_Register($src1$$reg),
13023               as_Register($src2$$reg),
13024               Assembler::ASR,
13025               $src3$$constant & 0x3f);
13026   %}
13027 
13028   ins_pipe(ialu_reg_reg_shift);
13029 %}
13030 
13031 // This pattern is automatically generated from aarch64_ad.m4
13032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13033 instruct SubI_reg_LShift_reg(iRegINoSp dst,
13034                          iRegIorL2I src1, iRegIorL2I src2,
13035                          immI src3) %{
13036   match(Set dst (SubI src1 (LShiftI src2 src3)));
13037 
13038   ins_cost(1.9 * INSN_COST);
13039   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
13040 
13041   ins_encode %{
13042     __ subw(as_Register($dst$$reg),
13043               as_Register($src1$$reg),
13044               as_Register($src2$$reg),
13045               Assembler::LSL,
13046               $src3$$constant & 0x1f);
13047   %}
13048 
13049   ins_pipe(ialu_reg_reg_shift);
13050 %}
13051 
13052 // This pattern is automatically generated from aarch64_ad.m4
13053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13054 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
13055                          iRegL src1, iRegL src2,
13056                          immI src3) %{
13057   match(Set dst (SubL src1 (LShiftL src2 src3)));
13058 
13059   ins_cost(1.9 * INSN_COST);
13060   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
13061 
13062   ins_encode %{
13063     __ sub(as_Register($dst$$reg),
13064               as_Register($src1$$reg),
13065               as_Register($src2$$reg),
13066               Assembler::LSL,
13067               $src3$$constant & 0x3f);
13068   %}
13069 
13070   ins_pipe(ialu_reg_reg_shift);
13071 %}
13072 
13073 // This pattern is automatically generated from aarch64_ad.m4
13074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13075 
13076 // Shift Left followed by Shift Right.
13077 // This idiom is used by the compiler for the i2b bytecode etc.
13078 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13079 %{
13080   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
13081   ins_cost(INSN_COST * 2);
13082   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13083   ins_encode %{
13084     int lshift = $lshift_count$$constant & 63;
13085     int rshift = $rshift_count$$constant & 63;
13086     int s = 63 - lshift;
13087     int r = (rshift - lshift) & 63;
13088     __ sbfm(as_Register($dst$$reg),
13089             as_Register($src$$reg),
13090             r, s);
13091   %}
13092 
13093   ins_pipe(ialu_reg_shift);
13094 %}
13095 
13096 // This pattern is automatically generated from aarch64_ad.m4
13097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13098 
13099 // Shift Left followed by Shift Right.
13100 // This idiom is used by the compiler for the i2b bytecode etc.
13101 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13102 %{
13103   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
13104   ins_cost(INSN_COST * 2);
13105   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13106   ins_encode %{
13107     int lshift = $lshift_count$$constant & 31;
13108     int rshift = $rshift_count$$constant & 31;
13109     int s = 31 - lshift;
13110     int r = (rshift - lshift) & 31;
13111     __ sbfmw(as_Register($dst$$reg),
13112             as_Register($src$$reg),
13113             r, s);
13114   %}
13115 
13116   ins_pipe(ialu_reg_shift);
13117 %}
13118 
13119 // This pattern is automatically generated from aarch64_ad.m4
13120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13121 
13122 // Shift Left followed by Shift Right.
13123 // This idiom is used by the compiler for the i2b bytecode etc.
13124 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13125 %{
13126   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
13127   ins_cost(INSN_COST * 2);
13128   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13129   ins_encode %{
13130     int lshift = $lshift_count$$constant & 63;
13131     int rshift = $rshift_count$$constant & 63;
13132     int s = 63 - lshift;
13133     int r = (rshift - lshift) & 63;
13134     __ ubfm(as_Register($dst$$reg),
13135             as_Register($src$$reg),
13136             r, s);
13137   %}
13138 
13139   ins_pipe(ialu_reg_shift);
13140 %}
13141 
13142 // This pattern is automatically generated from aarch64_ad.m4
13143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13144 
13145 // Shift Left followed by Shift Right.
13146 // This idiom is used by the compiler for the i2b bytecode etc.
13147 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13148 %{
13149   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13150   ins_cost(INSN_COST * 2);
13151   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13152   ins_encode %{
13153     int lshift = $lshift_count$$constant & 31;
13154     int rshift = $rshift_count$$constant & 31;
13155     int s = 31 - lshift;
13156     int r = (rshift - lshift) & 31;
13157     __ ubfmw(as_Register($dst$$reg),
13158             as_Register($src$$reg),
13159             r, s);
13160   %}
13161 
13162   ins_pipe(ialu_reg_shift);
13163 %}
13164 
13165 // Bitfield extract with shift & mask
13166 
13167 // This pattern is automatically generated from aarch64_ad.m4
13168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13169 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13170 %{
13171   match(Set dst (AndI (URShiftI src rshift) mask));
13172   // Make sure we are not going to exceed what ubfxw can do.
13173   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13174 
13175   ins_cost(INSN_COST);
13176   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13177   ins_encode %{
13178     int rshift = $rshift$$constant & 31;
13179     intptr_t mask = $mask$$constant;
13180     int width = exact_log2(mask+1);
13181     __ ubfxw(as_Register($dst$$reg),
13182             as_Register($src$$reg), rshift, width);
13183   %}
13184   ins_pipe(ialu_reg_shift);
13185 %}
13186 
13187 // This pattern is automatically generated from aarch64_ad.m4
13188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13189 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13190 %{
13191   match(Set dst (AndL (URShiftL src rshift) mask));
13192   // Make sure we are not going to exceed what ubfx can do.
13193   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13194 
13195   ins_cost(INSN_COST);
13196   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13197   ins_encode %{
13198     int rshift = $rshift$$constant & 63;
13199     intptr_t mask = $mask$$constant;
13200     int width = exact_log2_long(mask+1);
13201     __ ubfx(as_Register($dst$$reg),
13202             as_Register($src$$reg), rshift, width);
13203   %}
13204   ins_pipe(ialu_reg_shift);
13205 %}
13206 
13207 
13208 // This pattern is automatically generated from aarch64_ad.m4
13209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13210 
13211 // We can use ubfx when extending an And with a mask when we know mask
13212 // is positive.  We know that because immI_bitmask guarantees it.
13213 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13214 %{
13215   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13216   // Make sure we are not going to exceed what ubfxw can do.
13217   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13218 
13219   ins_cost(INSN_COST * 2);
13220   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13221   ins_encode %{
13222     int rshift = $rshift$$constant & 31;
13223     intptr_t mask = $mask$$constant;
13224     int width = exact_log2(mask+1);
13225     __ ubfx(as_Register($dst$$reg),
13226             as_Register($src$$reg), rshift, width);
13227   %}
13228   ins_pipe(ialu_reg_shift);
13229 %}
13230 
13231 
13232 // This pattern is automatically generated from aarch64_ad.m4
13233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13234 
13235 // We can use ubfiz when masking by a positive number and then left shifting the result.
13236 // We know that the mask is positive because immI_bitmask guarantees it.
13237 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13238 %{
13239   match(Set dst (LShiftI (AndI src mask) lshift));
13240   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13241 
13242   ins_cost(INSN_COST);
13243   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13244   ins_encode %{
13245     int lshift = $lshift$$constant & 31;
13246     intptr_t mask = $mask$$constant;
13247     int width = exact_log2(mask+1);
13248     __ ubfizw(as_Register($dst$$reg),
13249           as_Register($src$$reg), lshift, width);
13250   %}
13251   ins_pipe(ialu_reg_shift);
13252 %}
13253 
13254 // This pattern is automatically generated from aarch64_ad.m4
13255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13256 
13257 // We can use ubfiz when masking by a positive number and then left shifting the result.
13258 // We know that the mask is positive because immL_bitmask guarantees it.
13259 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13260 %{
13261   match(Set dst (LShiftL (AndL src mask) lshift));
13262   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13263 
13264   ins_cost(INSN_COST);
13265   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13266   ins_encode %{
13267     int lshift = $lshift$$constant & 63;
13268     intptr_t mask = $mask$$constant;
13269     int width = exact_log2_long(mask+1);
13270     __ ubfiz(as_Register($dst$$reg),
13271           as_Register($src$$reg), lshift, width);
13272   %}
13273   ins_pipe(ialu_reg_shift);
13274 %}
13275 
13276 // This pattern is automatically generated from aarch64_ad.m4
13277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13278 
13279 // We can use ubfiz when masking by a positive number and then left shifting the result.
13280 // We know that the mask is positive because immI_bitmask guarantees it.
13281 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13282 %{
13283   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13284   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13285 
13286   ins_cost(INSN_COST);
13287   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13288   ins_encode %{
13289     int lshift = $lshift$$constant & 31;
13290     intptr_t mask = $mask$$constant;
13291     int width = exact_log2(mask+1);
13292     __ ubfizw(as_Register($dst$$reg),
13293           as_Register($src$$reg), lshift, width);
13294   %}
13295   ins_pipe(ialu_reg_shift);
13296 %}
13297 
13298 // This pattern is automatically generated from aarch64_ad.m4
13299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13300 
13301 // We can use ubfiz when masking by a positive number and then left shifting the result.
13302 // We know that the mask is positive because immL_bitmask guarantees it.
13303 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13304 %{
13305   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13306   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13307 
13308   ins_cost(INSN_COST);
13309   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13310   ins_encode %{
13311     int lshift = $lshift$$constant & 63;
13312     intptr_t mask = $mask$$constant;
13313     int width = exact_log2_long(mask+1);
13314     __ ubfiz(as_Register($dst$$reg),
13315           as_Register($src$$reg), lshift, width);
13316   %}
13317   ins_pipe(ialu_reg_shift);
13318 %}
13319 
13320 
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323 
13324 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13325 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13326 %{
13327   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13328   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13329 
13330   ins_cost(INSN_COST);
13331   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13332   ins_encode %{
13333     int lshift = $lshift$$constant & 63;
13334     intptr_t mask = $mask$$constant;
13335     int width = exact_log2(mask+1);
13336     __ ubfiz(as_Register($dst$$reg),
13337              as_Register($src$$reg), lshift, width);
13338   %}
13339   ins_pipe(ialu_reg_shift);
13340 %}
13341 
13342 // This pattern is automatically generated from aarch64_ad.m4
13343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13344 
13345 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13346 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13347 %{
13348   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13349   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13350 
13351   ins_cost(INSN_COST);
13352   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13353   ins_encode %{
13354     int lshift = $lshift$$constant & 31;
13355     intptr_t mask = $mask$$constant;
13356     int width = exact_log2(mask+1);
13357     __ ubfiz(as_Register($dst$$reg),
13358              as_Register($src$$reg), lshift, width);
13359   %}
13360   ins_pipe(ialu_reg_shift);
13361 %}
13362 
13363 // This pattern is automatically generated from aarch64_ad.m4
13364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13365 
13366 // Can skip int2long conversions after AND with small bitmask
13367 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13368 %{
13369   match(Set dst (ConvI2L (AndI src msk)));
13370   ins_cost(INSN_COST);
13371   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13372   ins_encode %{
13373     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13374   %}
13375   ins_pipe(ialu_reg_shift);
13376 %}
13377 
13378 
13379 // Rotations
13380 // This pattern is automatically generated from aarch64_ad.m4
13381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13382 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13383 %{
13384   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13385   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13386 
13387   ins_cost(INSN_COST);
13388   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13389 
13390   ins_encode %{
13391     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13392             $rshift$$constant & 63);
13393   %}
13394   ins_pipe(ialu_reg_reg_extr);
13395 %}
13396 
13397 
13398 // This pattern is automatically generated from aarch64_ad.m4
13399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13400 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13401 %{
13402   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13403   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13404 
13405   ins_cost(INSN_COST);
13406   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13407 
13408   ins_encode %{
13409     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13410             $rshift$$constant & 31);
13411   %}
13412   ins_pipe(ialu_reg_reg_extr);
13413 %}
13414 
13415 
13416 // This pattern is automatically generated from aarch64_ad.m4
13417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13418 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13419 %{
13420   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13421   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13422 
13423   ins_cost(INSN_COST);
13424   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13425 
13426   ins_encode %{
13427     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13428             $rshift$$constant & 63);
13429   %}
13430   ins_pipe(ialu_reg_reg_extr);
13431 %}
13432 
13433 
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13437 %{
13438   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13439   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13440 
13441   ins_cost(INSN_COST);
13442   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13443 
13444   ins_encode %{
13445     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13446             $rshift$$constant & 31);
13447   %}
13448   ins_pipe(ialu_reg_reg_extr);
13449 %}
13450 
13451 
13452 // This pattern is automatically generated from aarch64_ad.m4
13453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13454 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13455 %{
13456   match(Set dst (RotateRight src shift));
13457 
13458   ins_cost(INSN_COST);
13459   format %{ "ror    $dst, $src, $shift" %}
13460 
13461   ins_encode %{
13462      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13463                $shift$$constant & 0x1f);
13464   %}
13465   ins_pipe(ialu_reg_reg_vshift);
13466 %}
13467 
13468 // This pattern is automatically generated from aarch64_ad.m4
13469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13470 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13471 %{
13472   match(Set dst (RotateRight src shift));
13473 
13474   ins_cost(INSN_COST);
13475   format %{ "ror    $dst, $src, $shift" %}
13476 
13477   ins_encode %{
13478      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13479                $shift$$constant & 0x3f);
13480   %}
13481   ins_pipe(ialu_reg_reg_vshift);
13482 %}
13483 
13484 // This pattern is automatically generated from aarch64_ad.m4
13485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13486 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13487 %{
13488   match(Set dst (RotateRight src shift));
13489 
13490   ins_cost(INSN_COST);
13491   format %{ "ror    $dst, $src, $shift" %}
13492 
13493   ins_encode %{
13494      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13495   %}
13496   ins_pipe(ialu_reg_reg_vshift);
13497 %}
13498 
13499 // This pattern is automatically generated from aarch64_ad.m4
13500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13501 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13502 %{
13503   match(Set dst (RotateRight src shift));
13504 
13505   ins_cost(INSN_COST);
13506   format %{ "ror    $dst, $src, $shift" %}
13507 
13508   ins_encode %{
13509      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13510   %}
13511   ins_pipe(ialu_reg_reg_vshift);
13512 %}
13513 
13514 // This pattern is automatically generated from aarch64_ad.m4
13515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13516 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13517 %{
13518   match(Set dst (RotateLeft src shift));
13519 
13520   ins_cost(INSN_COST);
13521   format %{ "rol    $dst, $src, $shift" %}
13522 
13523   ins_encode %{
13524      __ subw(rscratch1, zr, as_Register($shift$$reg));
13525      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13526   %}
13527   ins_pipe(ialu_reg_reg_vshift);
13528 %}
13529 
13530 // This pattern is automatically generated from aarch64_ad.m4
13531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13532 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13533 %{
13534   match(Set dst (RotateLeft src shift));
13535 
13536   ins_cost(INSN_COST);
13537   format %{ "rol    $dst, $src, $shift" %}
13538 
13539   ins_encode %{
13540      __ subw(rscratch1, zr, as_Register($shift$$reg));
13541      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13542   %}
13543   ins_pipe(ialu_reg_reg_vshift);
13544 %}
13545 
13546 
13547 // Add/subtract (extended)
13548 
13549 // This pattern is automatically generated from aarch64_ad.m4
13550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13551 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13552 %{
13553   match(Set dst (AddL src1 (ConvI2L src2)));
13554   ins_cost(INSN_COST);
13555   format %{ "add  $dst, $src1, $src2, sxtw" %}
13556 
13557    ins_encode %{
13558      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13559             as_Register($src2$$reg), ext::sxtw);
13560    %}
13561   ins_pipe(ialu_reg_reg);
13562 %}
13563 
13564 // This pattern is automatically generated from aarch64_ad.m4
13565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13566 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13567 %{
13568   match(Set dst (SubL src1 (ConvI2L src2)));
13569   ins_cost(INSN_COST);
13570   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13571 
13572    ins_encode %{
13573      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13574             as_Register($src2$$reg), ext::sxtw);
13575    %}
13576   ins_pipe(ialu_reg_reg);
13577 %}
13578 
13579 // This pattern is automatically generated from aarch64_ad.m4
13580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13581 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13582 %{
13583   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13584   ins_cost(INSN_COST);
13585   format %{ "add  $dst, $src1, $src2, sxth" %}
13586 
13587    ins_encode %{
13588      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13589             as_Register($src2$$reg), ext::sxth);
13590    %}
13591   ins_pipe(ialu_reg_reg);
13592 %}
13593 
13594 // This pattern is automatically generated from aarch64_ad.m4
13595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13596 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13597 %{
13598   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13599   ins_cost(INSN_COST);
13600   format %{ "add  $dst, $src1, $src2, sxtb" %}
13601 
13602    ins_encode %{
13603      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13604             as_Register($src2$$reg), ext::sxtb);
13605    %}
13606   ins_pipe(ialu_reg_reg);
13607 %}
13608 
13609 // This pattern is automatically generated from aarch64_ad.m4
13610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13611 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13612 %{
13613   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13614   ins_cost(INSN_COST);
13615   format %{ "add  $dst, $src1, $src2, uxtb" %}
13616 
13617    ins_encode %{
13618      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13619             as_Register($src2$$reg), ext::uxtb);
13620    %}
13621   ins_pipe(ialu_reg_reg);
13622 %}
13623 
13624 // This pattern is automatically generated from aarch64_ad.m4
13625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13626 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13627 %{
13628   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13629   ins_cost(INSN_COST);
13630   format %{ "add  $dst, $src1, $src2, sxth" %}
13631 
13632    ins_encode %{
13633      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13634             as_Register($src2$$reg), ext::sxth);
13635    %}
13636   ins_pipe(ialu_reg_reg);
13637 %}
13638 
13639 // This pattern is automatically generated from aarch64_ad.m4
13640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13641 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13642 %{
13643   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13644   ins_cost(INSN_COST);
13645   format %{ "add  $dst, $src1, $src2, sxtw" %}
13646 
13647    ins_encode %{
13648      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13649             as_Register($src2$$reg), ext::sxtw);
13650    %}
13651   ins_pipe(ialu_reg_reg);
13652 %}
13653 
13654 // This pattern is automatically generated from aarch64_ad.m4
13655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13656 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13657 %{
13658   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13659   ins_cost(INSN_COST);
13660   format %{ "add  $dst, $src1, $src2, sxtb" %}
13661 
13662    ins_encode %{
13663      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13664             as_Register($src2$$reg), ext::sxtb);
13665    %}
13666   ins_pipe(ialu_reg_reg);
13667 %}
13668 
13669 // This pattern is automatically generated from aarch64_ad.m4
13670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13671 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13672 %{
13673   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13674   ins_cost(INSN_COST);
13675   format %{ "add  $dst, $src1, $src2, uxtb" %}
13676 
13677    ins_encode %{
13678      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13679             as_Register($src2$$reg), ext::uxtb);
13680    %}
13681   ins_pipe(ialu_reg_reg);
13682 %}
13683 
13684 // This pattern is automatically generated from aarch64_ad.m4
13685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13686 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13687 %{
13688   match(Set dst (AddI src1 (AndI src2 mask)));
13689   ins_cost(INSN_COST);
13690   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13691 
13692    ins_encode %{
13693      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13694             as_Register($src2$$reg), ext::uxtb);
13695    %}
13696   ins_pipe(ialu_reg_reg);
13697 %}
13698 
13699 // This pattern is automatically generated from aarch64_ad.m4
13700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13701 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13702 %{
13703   match(Set dst (AddI src1 (AndI src2 mask)));
13704   ins_cost(INSN_COST);
13705   format %{ "addw  $dst, $src1, $src2, uxth" %}
13706 
13707    ins_encode %{
13708      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13709             as_Register($src2$$reg), ext::uxth);
13710    %}
13711   ins_pipe(ialu_reg_reg);
13712 %}
13713 
13714 // This pattern is automatically generated from aarch64_ad.m4
13715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13716 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13717 %{
13718   match(Set dst (AddL src1 (AndL src2 mask)));
13719   ins_cost(INSN_COST);
13720   format %{ "add  $dst, $src1, $src2, uxtb" %}
13721 
13722    ins_encode %{
13723      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13724             as_Register($src2$$reg), ext::uxtb);
13725    %}
13726   ins_pipe(ialu_reg_reg);
13727 %}
13728 
13729 // This pattern is automatically generated from aarch64_ad.m4
13730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13731 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13732 %{
13733   match(Set dst (AddL src1 (AndL src2 mask)));
13734   ins_cost(INSN_COST);
13735   format %{ "add  $dst, $src1, $src2, uxth" %}
13736 
13737    ins_encode %{
13738      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13739             as_Register($src2$$reg), ext::uxth);
13740    %}
13741   ins_pipe(ialu_reg_reg);
13742 %}
13743 
13744 // This pattern is automatically generated from aarch64_ad.m4
13745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13746 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13747 %{
13748   match(Set dst (AddL src1 (AndL src2 mask)));
13749   ins_cost(INSN_COST);
13750   format %{ "add  $dst, $src1, $src2, uxtw" %}
13751 
13752    ins_encode %{
13753      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13754             as_Register($src2$$reg), ext::uxtw);
13755    %}
13756   ins_pipe(ialu_reg_reg);
13757 %}
13758 
13759 // This pattern is automatically generated from aarch64_ad.m4
13760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13761 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13762 %{
13763   match(Set dst (SubI src1 (AndI src2 mask)));
13764   ins_cost(INSN_COST);
13765   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13766 
13767    ins_encode %{
13768      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13769             as_Register($src2$$reg), ext::uxtb);
13770    %}
13771   ins_pipe(ialu_reg_reg);
13772 %}
13773 
13774 // This pattern is automatically generated from aarch64_ad.m4
13775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13776 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13777 %{
13778   match(Set dst (SubI src1 (AndI src2 mask)));
13779   ins_cost(INSN_COST);
13780   format %{ "subw  $dst, $src1, $src2, uxth" %}
13781 
13782    ins_encode %{
13783      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13784             as_Register($src2$$reg), ext::uxth);
13785    %}
13786   ins_pipe(ialu_reg_reg);
13787 %}
13788 
13789 // This pattern is automatically generated from aarch64_ad.m4
13790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13791 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13792 %{
13793   match(Set dst (SubL src1 (AndL src2 mask)));
13794   ins_cost(INSN_COST);
13795   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13796 
13797    ins_encode %{
13798      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13799             as_Register($src2$$reg), ext::uxtb);
13800    %}
13801   ins_pipe(ialu_reg_reg);
13802 %}
13803 
13804 // This pattern is automatically generated from aarch64_ad.m4
13805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13806 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13807 %{
13808   match(Set dst (SubL src1 (AndL src2 mask)));
13809   ins_cost(INSN_COST);
13810   format %{ "sub  $dst, $src1, $src2, uxth" %}
13811 
13812    ins_encode %{
13813      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13814             as_Register($src2$$reg), ext::uxth);
13815    %}
13816   ins_pipe(ialu_reg_reg);
13817 %}
13818 
13819 // This pattern is automatically generated from aarch64_ad.m4
13820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13821 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13822 %{
13823   match(Set dst (SubL src1 (AndL src2 mask)));
13824   ins_cost(INSN_COST);
13825   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13826 
13827    ins_encode %{
13828      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13829             as_Register($src2$$reg), ext::uxtw);
13830    %}
13831   ins_pipe(ialu_reg_reg);
13832 %}
13833 
13834 
13835 // This pattern is automatically generated from aarch64_ad.m4
13836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13837 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13838 %{
13839   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13840   ins_cost(1.9 * INSN_COST);
13841   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13842 
13843    ins_encode %{
13844      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13845             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13846    %}
13847   ins_pipe(ialu_reg_reg_shift);
13848 %}
13849 
13850 // This pattern is automatically generated from aarch64_ad.m4
13851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13852 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13853 %{
13854   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13855   ins_cost(1.9 * INSN_COST);
13856   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13857 
13858    ins_encode %{
13859      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13860             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13861    %}
13862   ins_pipe(ialu_reg_reg_shift);
13863 %}
13864 
13865 // This pattern is automatically generated from aarch64_ad.m4
13866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13867 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13868 %{
13869   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13870   ins_cost(1.9 * INSN_COST);
13871   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13872 
13873    ins_encode %{
13874      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13875             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13876    %}
13877   ins_pipe(ialu_reg_reg_shift);
13878 %}
13879 
13880 // This pattern is automatically generated from aarch64_ad.m4
13881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13882 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13883 %{
13884   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13885   ins_cost(1.9 * INSN_COST);
13886   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13887 
13888    ins_encode %{
13889      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13890             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13891    %}
13892   ins_pipe(ialu_reg_reg_shift);
13893 %}
13894 
13895 // This pattern is automatically generated from aarch64_ad.m4
13896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13897 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13898 %{
13899   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13900   ins_cost(1.9 * INSN_COST);
13901   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13902 
13903    ins_encode %{
13904      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13905             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13906    %}
13907   ins_pipe(ialu_reg_reg_shift);
13908 %}
13909 
13910 // This pattern is automatically generated from aarch64_ad.m4
13911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13912 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13913 %{
13914   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13915   ins_cost(1.9 * INSN_COST);
13916   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13917 
13918    ins_encode %{
13919      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13920             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13921    %}
13922   ins_pipe(ialu_reg_reg_shift);
13923 %}
13924 
13925 // This pattern is automatically generated from aarch64_ad.m4
13926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13927 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13928 %{
13929   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13930   ins_cost(1.9 * INSN_COST);
13931   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13932 
13933    ins_encode %{
13934      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13935             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13936    %}
13937   ins_pipe(ialu_reg_reg_shift);
13938 %}
13939 
13940 // This pattern is automatically generated from aarch64_ad.m4
13941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13942 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13943 %{
13944   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13945   ins_cost(1.9 * INSN_COST);
13946   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13947 
13948    ins_encode %{
13949      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13950             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13951    %}
13952   ins_pipe(ialu_reg_reg_shift);
13953 %}
13954 
13955 // This pattern is automatically generated from aarch64_ad.m4
13956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13957 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13958 %{
13959   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13960   ins_cost(1.9 * INSN_COST);
13961   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13962 
13963    ins_encode %{
13964      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13965             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13966    %}
13967   ins_pipe(ialu_reg_reg_shift);
13968 %}
13969 
13970 // This pattern is automatically generated from aarch64_ad.m4
13971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13972 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13973 %{
13974   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13975   ins_cost(1.9 * INSN_COST);
13976   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13977 
13978    ins_encode %{
13979      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13980             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13981    %}
13982   ins_pipe(ialu_reg_reg_shift);
13983 %}
13984 
13985 // This pattern is automatically generated from aarch64_ad.m4
13986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13987 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13988 %{
13989   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13990   ins_cost(1.9 * INSN_COST);
13991   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13992 
13993    ins_encode %{
13994      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13995             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13996    %}
13997   ins_pipe(ialu_reg_reg_shift);
13998 %}
13999 
14000 // This pattern is automatically generated from aarch64_ad.m4
14001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14002 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
14003 %{
14004   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
14005   ins_cost(1.9 * INSN_COST);
14006   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
14007 
14008    ins_encode %{
14009      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14010             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
14011    %}
14012   ins_pipe(ialu_reg_reg_shift);
14013 %}
14014 
14015 // This pattern is automatically generated from aarch64_ad.m4
14016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14017 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14018 %{
14019   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14020   ins_cost(1.9 * INSN_COST);
14021   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
14022 
14023    ins_encode %{
14024      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14025             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14026    %}
14027   ins_pipe(ialu_reg_reg_shift);
14028 %}
14029 
14030 // This pattern is automatically generated from aarch64_ad.m4
14031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14032 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14033 %{
14034   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14035   ins_cost(1.9 * INSN_COST);
14036   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
14037 
14038    ins_encode %{
14039      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14040             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14041    %}
14042   ins_pipe(ialu_reg_reg_shift);
14043 %}
14044 
14045 // This pattern is automatically generated from aarch64_ad.m4
14046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14047 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14048 %{
14049   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14050   ins_cost(1.9 * INSN_COST);
14051   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
14052 
14053    ins_encode %{
14054      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14055             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14056    %}
14057   ins_pipe(ialu_reg_reg_shift);
14058 %}
14059 
14060 // This pattern is automatically generated from aarch64_ad.m4
14061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14062 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14063 %{
14064   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14065   ins_cost(1.9 * INSN_COST);
14066   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
14067 
14068    ins_encode %{
14069      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14070             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14071    %}
14072   ins_pipe(ialu_reg_reg_shift);
14073 %}
14074 
14075 // This pattern is automatically generated from aarch64_ad.m4
14076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14077 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14078 %{
14079   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14080   ins_cost(1.9 * INSN_COST);
14081   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
14082 
14083    ins_encode %{
14084      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14085             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14086    %}
14087   ins_pipe(ialu_reg_reg_shift);
14088 %}
14089 
14090 // This pattern is automatically generated from aarch64_ad.m4
14091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14092 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14093 %{
14094   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14095   ins_cost(1.9 * INSN_COST);
14096   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
14097 
14098    ins_encode %{
14099      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14100             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14101    %}
14102   ins_pipe(ialu_reg_reg_shift);
14103 %}
14104 
14105 // This pattern is automatically generated from aarch64_ad.m4
14106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14107 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14108 %{
14109   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14110   ins_cost(1.9 * INSN_COST);
14111   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
14112 
14113    ins_encode %{
14114      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14115             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14116    %}
14117   ins_pipe(ialu_reg_reg_shift);
14118 %}
14119 
14120 // This pattern is automatically generated from aarch64_ad.m4
14121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14122 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14123 %{
14124   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14125   ins_cost(1.9 * INSN_COST);
14126   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
14127 
14128    ins_encode %{
14129      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14130             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14131    %}
14132   ins_pipe(ialu_reg_reg_shift);
14133 %}
14134 
14135 // This pattern is automatically generated from aarch64_ad.m4
14136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14137 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14138 %{
14139   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14140   ins_cost(1.9 * INSN_COST);
14141   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
14142 
14143    ins_encode %{
14144      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14145             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14146    %}
14147   ins_pipe(ialu_reg_reg_shift);
14148 %}
14149 
14150 // This pattern is automatically generated from aarch64_ad.m4
14151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14152 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14153 %{
14154   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14155   ins_cost(1.9 * INSN_COST);
14156   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
14157 
14158    ins_encode %{
14159      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14160             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14161    %}
14162   ins_pipe(ialu_reg_reg_shift);
14163 %}
14164 
14165 
14166 
14167 // END This section of the file is automatically generated. Do not edit --------------
14168 
14169 
14170 // ============================================================================
14171 // Floating Point Arithmetic Instructions
14172 
14173 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14174   match(Set dst (AddF src1 src2));
14175 
14176   ins_cost(INSN_COST * 5);
14177   format %{ "fadds   $dst, $src1, $src2" %}
14178 
14179   ins_encode %{
14180     __ fadds(as_FloatRegister($dst$$reg),
14181              as_FloatRegister($src1$$reg),
14182              as_FloatRegister($src2$$reg));
14183   %}
14184 
14185   ins_pipe(fp_dop_reg_reg_s);
14186 %}
14187 
14188 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14189   match(Set dst (AddD src1 src2));
14190 
14191   ins_cost(INSN_COST * 5);
14192   format %{ "faddd   $dst, $src1, $src2" %}
14193 
14194   ins_encode %{
14195     __ faddd(as_FloatRegister($dst$$reg),
14196              as_FloatRegister($src1$$reg),
14197              as_FloatRegister($src2$$reg));
14198   %}
14199 
14200   ins_pipe(fp_dop_reg_reg_d);
14201 %}
14202 
14203 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14204   match(Set dst (SubF src1 src2));
14205 
14206   ins_cost(INSN_COST * 5);
14207   format %{ "fsubs   $dst, $src1, $src2" %}
14208 
14209   ins_encode %{
14210     __ fsubs(as_FloatRegister($dst$$reg),
14211              as_FloatRegister($src1$$reg),
14212              as_FloatRegister($src2$$reg));
14213   %}
14214 
14215   ins_pipe(fp_dop_reg_reg_s);
14216 %}
14217 
14218 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14219   match(Set dst (SubD src1 src2));
14220 
14221   ins_cost(INSN_COST * 5);
14222   format %{ "fsubd   $dst, $src1, $src2" %}
14223 
14224   ins_encode %{
14225     __ fsubd(as_FloatRegister($dst$$reg),
14226              as_FloatRegister($src1$$reg),
14227              as_FloatRegister($src2$$reg));
14228   %}
14229 
14230   ins_pipe(fp_dop_reg_reg_d);
14231 %}
14232 
14233 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14234   match(Set dst (MulF src1 src2));
14235 
14236   ins_cost(INSN_COST * 6);
14237   format %{ "fmuls   $dst, $src1, $src2" %}
14238 
14239   ins_encode %{
14240     __ fmuls(as_FloatRegister($dst$$reg),
14241              as_FloatRegister($src1$$reg),
14242              as_FloatRegister($src2$$reg));
14243   %}
14244 
14245   ins_pipe(fp_dop_reg_reg_s);
14246 %}
14247 
14248 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14249   match(Set dst (MulD src1 src2));
14250 
14251   ins_cost(INSN_COST * 6);
14252   format %{ "fmuld   $dst, $src1, $src2" %}
14253 
14254   ins_encode %{
14255     __ fmuld(as_FloatRegister($dst$$reg),
14256              as_FloatRegister($src1$$reg),
14257              as_FloatRegister($src2$$reg));
14258   %}
14259 
14260   ins_pipe(fp_dop_reg_reg_d);
14261 %}
14262 
14263 // src1 * src2 + src3
14264 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14265   predicate(UseFMA);
14266   match(Set dst (FmaF src3 (Binary src1 src2)));
14267 
14268   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14269 
14270   ins_encode %{
14271     __ fmadds(as_FloatRegister($dst$$reg),
14272              as_FloatRegister($src1$$reg),
14273              as_FloatRegister($src2$$reg),
14274              as_FloatRegister($src3$$reg));
14275   %}
14276 
14277   ins_pipe(pipe_class_default);
14278 %}
14279 
14280 // src1 * src2 + src3
14281 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14282   predicate(UseFMA);
14283   match(Set dst (FmaD src3 (Binary src1 src2)));
14284 
14285   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14286 
14287   ins_encode %{
14288     __ fmaddd(as_FloatRegister($dst$$reg),
14289              as_FloatRegister($src1$$reg),
14290              as_FloatRegister($src2$$reg),
14291              as_FloatRegister($src3$$reg));
14292   %}
14293 
14294   ins_pipe(pipe_class_default);
14295 %}
14296 
14297 // -src1 * src2 + src3
14298 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14299   predicate(UseFMA);
14300   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14301   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14302 
14303   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14304 
14305   ins_encode %{
14306     __ fmsubs(as_FloatRegister($dst$$reg),
14307               as_FloatRegister($src1$$reg),
14308               as_FloatRegister($src2$$reg),
14309               as_FloatRegister($src3$$reg));
14310   %}
14311 
14312   ins_pipe(pipe_class_default);
14313 %}
14314 
14315 // -src1 * src2 + src3
14316 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14317   predicate(UseFMA);
14318   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14319   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14320 
14321   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14322 
14323   ins_encode %{
14324     __ fmsubd(as_FloatRegister($dst$$reg),
14325               as_FloatRegister($src1$$reg),
14326               as_FloatRegister($src2$$reg),
14327               as_FloatRegister($src3$$reg));
14328   %}
14329 
14330   ins_pipe(pipe_class_default);
14331 %}
14332 
14333 // -src1 * src2 - src3
14334 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14335   predicate(UseFMA);
14336   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14337   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14338 
14339   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14340 
14341   ins_encode %{
14342     __ fnmadds(as_FloatRegister($dst$$reg),
14343                as_FloatRegister($src1$$reg),
14344                as_FloatRegister($src2$$reg),
14345                as_FloatRegister($src3$$reg));
14346   %}
14347 
14348   ins_pipe(pipe_class_default);
14349 %}
14350 
14351 // -src1 * src2 - src3
14352 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14353   predicate(UseFMA);
14354   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14355   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14356 
14357   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14358 
14359   ins_encode %{
14360     __ fnmaddd(as_FloatRegister($dst$$reg),
14361                as_FloatRegister($src1$$reg),
14362                as_FloatRegister($src2$$reg),
14363                as_FloatRegister($src3$$reg));
14364   %}
14365 
14366   ins_pipe(pipe_class_default);
14367 %}
14368 
14369 // src1 * src2 - src3
14370 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14371   predicate(UseFMA);
14372   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14373 
14374   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14375 
14376   ins_encode %{
14377     __ fnmsubs(as_FloatRegister($dst$$reg),
14378                as_FloatRegister($src1$$reg),
14379                as_FloatRegister($src2$$reg),
14380                as_FloatRegister($src3$$reg));
14381   %}
14382 
14383   ins_pipe(pipe_class_default);
14384 %}
14385 
14386 // src1 * src2 - src3
14387 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14388   predicate(UseFMA);
14389   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14390 
14391   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14392 
14393   ins_encode %{
14394   // n.b. insn name should be fnmsubd
14395     __ fnmsub(as_FloatRegister($dst$$reg),
14396               as_FloatRegister($src1$$reg),
14397               as_FloatRegister($src2$$reg),
14398               as_FloatRegister($src3$$reg));
14399   %}
14400 
14401   ins_pipe(pipe_class_default);
14402 %}
14403 
14404 
14405 // Math.max(FF)F
14406 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14407   match(Set dst (MaxF src1 src2));
14408 
14409   format %{ "fmaxs   $dst, $src1, $src2" %}
14410   ins_encode %{
14411     __ fmaxs(as_FloatRegister($dst$$reg),
14412              as_FloatRegister($src1$$reg),
14413              as_FloatRegister($src2$$reg));
14414   %}
14415 
14416   ins_pipe(fp_dop_reg_reg_s);
14417 %}
14418 
14419 // Math.min(FF)F
14420 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14421   match(Set dst (MinF src1 src2));
14422 
14423   format %{ "fmins   $dst, $src1, $src2" %}
14424   ins_encode %{
14425     __ fmins(as_FloatRegister($dst$$reg),
14426              as_FloatRegister($src1$$reg),
14427              as_FloatRegister($src2$$reg));
14428   %}
14429 
14430   ins_pipe(fp_dop_reg_reg_s);
14431 %}
14432 
14433 // Math.max(DD)D
14434 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14435   match(Set dst (MaxD src1 src2));
14436 
14437   format %{ "fmaxd   $dst, $src1, $src2" %}
14438   ins_encode %{
14439     __ fmaxd(as_FloatRegister($dst$$reg),
14440              as_FloatRegister($src1$$reg),
14441              as_FloatRegister($src2$$reg));
14442   %}
14443 
14444   ins_pipe(fp_dop_reg_reg_d);
14445 %}
14446 
14447 // Math.min(DD)D
14448 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14449   match(Set dst (MinD src1 src2));
14450 
14451   format %{ "fmind   $dst, $src1, $src2" %}
14452   ins_encode %{
14453     __ fmind(as_FloatRegister($dst$$reg),
14454              as_FloatRegister($src1$$reg),
14455              as_FloatRegister($src2$$reg));
14456   %}
14457 
14458   ins_pipe(fp_dop_reg_reg_d);
14459 %}
14460 
14461 
14462 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14463   match(Set dst (DivF src1  src2));
14464 
14465   ins_cost(INSN_COST * 18);
14466   format %{ "fdivs   $dst, $src1, $src2" %}
14467 
14468   ins_encode %{
14469     __ fdivs(as_FloatRegister($dst$$reg),
14470              as_FloatRegister($src1$$reg),
14471              as_FloatRegister($src2$$reg));
14472   %}
14473 
14474   ins_pipe(fp_div_s);
14475 %}
14476 
14477 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14478   match(Set dst (DivD src1  src2));
14479 
14480   ins_cost(INSN_COST * 32);
14481   format %{ "fdivd   $dst, $src1, $src2" %}
14482 
14483   ins_encode %{
14484     __ fdivd(as_FloatRegister($dst$$reg),
14485              as_FloatRegister($src1$$reg),
14486              as_FloatRegister($src2$$reg));
14487   %}
14488 
14489   ins_pipe(fp_div_d);
14490 %}
14491 
14492 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14493   match(Set dst (NegF src));
14494 
14495   ins_cost(INSN_COST * 3);
14496   format %{ "fneg   $dst, $src" %}
14497 
14498   ins_encode %{
14499     __ fnegs(as_FloatRegister($dst$$reg),
14500              as_FloatRegister($src$$reg));
14501   %}
14502 
14503   ins_pipe(fp_uop_s);
14504 %}
14505 
14506 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14507   match(Set dst (NegD src));
14508 
14509   ins_cost(INSN_COST * 3);
14510   format %{ "fnegd   $dst, $src" %}
14511 
14512   ins_encode %{
14513     __ fnegd(as_FloatRegister($dst$$reg),
14514              as_FloatRegister($src$$reg));
14515   %}
14516 
14517   ins_pipe(fp_uop_d);
14518 %}
14519 
14520 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14521 %{
14522   match(Set dst (AbsI src));
14523 
14524   effect(KILL cr);
14525   ins_cost(INSN_COST * 2);
14526   format %{ "cmpw  $src, zr\n\t"
14527             "cnegw $dst, $src, Assembler::LT\t# int abs"
14528   %}
14529 
14530   ins_encode %{
14531     __ cmpw(as_Register($src$$reg), zr);
14532     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14533   %}
14534   ins_pipe(pipe_class_default);
14535 %}
14536 
14537 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14538 %{
14539   match(Set dst (AbsL src));
14540 
14541   effect(KILL cr);
14542   ins_cost(INSN_COST * 2);
14543   format %{ "cmp  $src, zr\n\t"
14544             "cneg $dst, $src, Assembler::LT\t# long abs"
14545   %}
14546 
14547   ins_encode %{
14548     __ cmp(as_Register($src$$reg), zr);
14549     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14550   %}
14551   ins_pipe(pipe_class_default);
14552 %}
14553 
14554 instruct absF_reg(vRegF dst, vRegF src) %{
14555   match(Set dst (AbsF src));
14556 
14557   ins_cost(INSN_COST * 3);
14558   format %{ "fabss   $dst, $src" %}
14559   ins_encode %{
14560     __ fabss(as_FloatRegister($dst$$reg),
14561              as_FloatRegister($src$$reg));
14562   %}
14563 
14564   ins_pipe(fp_uop_s);
14565 %}
14566 
14567 instruct absD_reg(vRegD dst, vRegD src) %{
14568   match(Set dst (AbsD src));
14569 
14570   ins_cost(INSN_COST * 3);
14571   format %{ "fabsd   $dst, $src" %}
14572   ins_encode %{
14573     __ fabsd(as_FloatRegister($dst$$reg),
14574              as_FloatRegister($src$$reg));
14575   %}
14576 
14577   ins_pipe(fp_uop_d);
14578 %}
14579 
14580 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14581   match(Set dst (AbsF (SubF src1 src2)));
14582 
14583   ins_cost(INSN_COST * 3);
14584   format %{ "fabds   $dst, $src1, $src2" %}
14585   ins_encode %{
14586     __ fabds(as_FloatRegister($dst$$reg),
14587              as_FloatRegister($src1$$reg),
14588              as_FloatRegister($src2$$reg));
14589   %}
14590 
14591   ins_pipe(fp_uop_s);
14592 %}
14593 
14594 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14595   match(Set dst (AbsD (SubD src1 src2)));
14596 
14597   ins_cost(INSN_COST * 3);
14598   format %{ "fabdd   $dst, $src1, $src2" %}
14599   ins_encode %{
14600     __ fabdd(as_FloatRegister($dst$$reg),
14601              as_FloatRegister($src1$$reg),
14602              as_FloatRegister($src2$$reg));
14603   %}
14604 
14605   ins_pipe(fp_uop_d);
14606 %}
14607 
14608 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14609   match(Set dst (SqrtD src));
14610 
14611   ins_cost(INSN_COST * 50);
14612   format %{ "fsqrtd  $dst, $src" %}
14613   ins_encode %{
14614     __ fsqrtd(as_FloatRegister($dst$$reg),
14615              as_FloatRegister($src$$reg));
14616   %}
14617 
14618   ins_pipe(fp_div_s);
14619 %}
14620 
14621 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14622   match(Set dst (SqrtF src));
14623 
14624   ins_cost(INSN_COST * 50);
14625   format %{ "fsqrts  $dst, $src" %}
14626   ins_encode %{
14627     __ fsqrts(as_FloatRegister($dst$$reg),
14628              as_FloatRegister($src$$reg));
14629   %}
14630 
14631   ins_pipe(fp_div_d);
14632 %}
14633 
14634 // Math.rint, floor, ceil
14635 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14636   match(Set dst (RoundDoubleMode src rmode));
14637   format %{ "frint  $dst, $src, $rmode" %}
14638   ins_encode %{
14639     switch ($rmode$$constant) {
14640       case RoundDoubleModeNode::rmode_rint:
14641         __ frintnd(as_FloatRegister($dst$$reg),
14642                    as_FloatRegister($src$$reg));
14643         break;
14644       case RoundDoubleModeNode::rmode_floor:
14645         __ frintmd(as_FloatRegister($dst$$reg),
14646                    as_FloatRegister($src$$reg));
14647         break;
14648       case RoundDoubleModeNode::rmode_ceil:
14649         __ frintpd(as_FloatRegister($dst$$reg),
14650                    as_FloatRegister($src$$reg));
14651         break;
14652     }
14653   %}
14654   ins_pipe(fp_uop_d);
14655 %}
14656 
14657 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14658   match(Set dst (CopySignD src1 (Binary src2 zero)));
14659   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14660   format %{ "CopySignD  $dst $src1 $src2" %}
14661   ins_encode %{
14662     FloatRegister dst = as_FloatRegister($dst$$reg),
14663                   src1 = as_FloatRegister($src1$$reg),
14664                   src2 = as_FloatRegister($src2$$reg),
14665                   zero = as_FloatRegister($zero$$reg);
14666     __ fnegd(dst, zero);
14667     __ bsl(dst, __ T8B, src2, src1);
14668   %}
14669   ins_pipe(fp_uop_d);
14670 %}
14671 
14672 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14673   match(Set dst (CopySignF src1 src2));
14674   effect(TEMP_DEF dst, USE src1, USE src2);
14675   format %{ "CopySignF  $dst $src1 $src2" %}
14676   ins_encode %{
14677     FloatRegister dst = as_FloatRegister($dst$$reg),
14678                   src1 = as_FloatRegister($src1$$reg),
14679                   src2 = as_FloatRegister($src2$$reg);
14680     __ movi(dst, __ T2S, 0x80, 24);
14681     __ bsl(dst, __ T8B, src2, src1);
14682   %}
14683   ins_pipe(fp_uop_d);
14684 %}
14685 
14686 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14687   match(Set dst (SignumD src (Binary zero one)));
14688   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14689   format %{ "signumD  $dst, $src" %}
14690   ins_encode %{
14691     FloatRegister src = as_FloatRegister($src$$reg),
14692                   dst = as_FloatRegister($dst$$reg),
14693                   zero = as_FloatRegister($zero$$reg),
14694                   one = as_FloatRegister($one$$reg);
14695     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14696     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14697     // Bit selection instruction gets bit from "one" for each enabled bit in
14698     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14699     // NaN the whole "src" will be copied because "dst" is zero. For all other
14700     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14701     // from "src", and all other bits are copied from 1.0.
14702     __ bsl(dst, __ T8B, one, src);
14703   %}
14704   ins_pipe(fp_uop_d);
14705 %}
14706 
14707 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14708   match(Set dst (SignumF src (Binary zero one)));
14709   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14710   format %{ "signumF  $dst, $src" %}
14711   ins_encode %{
14712     FloatRegister src = as_FloatRegister($src$$reg),
14713                   dst = as_FloatRegister($dst$$reg),
14714                   zero = as_FloatRegister($zero$$reg),
14715                   one = as_FloatRegister($one$$reg);
14716     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14717     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14718     // Bit selection instruction gets bit from "one" for each enabled bit in
14719     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14720     // NaN the whole "src" will be copied because "dst" is zero. For all other
14721     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14722     // from "src", and all other bits are copied from 1.0.
14723     __ bsl(dst, __ T8B, one, src);
14724   %}
14725   ins_pipe(fp_uop_d);
14726 %}
14727 
14728 instruct onspinwait() %{
14729   match(OnSpinWait);
14730   ins_cost(INSN_COST);
14731 
14732   format %{ "onspinwait" %}
14733 
14734   ins_encode %{
14735     __ spin_wait();
14736   %}
14737   ins_pipe(pipe_class_empty);
14738 %}
14739 
14740 // ============================================================================
14741 // Logical Instructions
14742 
14743 // Integer Logical Instructions
14744 
14745 // And Instructions
14746 
14747 
14748 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14749   match(Set dst (AndI src1 src2));
14750 
14751   format %{ "andw  $dst, $src1, $src2\t# int" %}
14752 
14753   ins_cost(INSN_COST);
14754   ins_encode %{
14755     __ andw(as_Register($dst$$reg),
14756             as_Register($src1$$reg),
14757             as_Register($src2$$reg));
14758   %}
14759 
14760   ins_pipe(ialu_reg_reg);
14761 %}
14762 
14763 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14764   match(Set dst (AndI src1 src2));
14765 
14766   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14767 
14768   ins_cost(INSN_COST);
14769   ins_encode %{
14770     __ andw(as_Register($dst$$reg),
14771             as_Register($src1$$reg),
14772             (uint64_t)($src2$$constant));
14773   %}
14774 
14775   ins_pipe(ialu_reg_imm);
14776 %}
14777 
14778 // Or Instructions
14779 
14780 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14781   match(Set dst (OrI src1 src2));
14782 
14783   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14784 
14785   ins_cost(INSN_COST);
14786   ins_encode %{
14787     __ orrw(as_Register($dst$$reg),
14788             as_Register($src1$$reg),
14789             as_Register($src2$$reg));
14790   %}
14791 
14792   ins_pipe(ialu_reg_reg);
14793 %}
14794 
14795 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14796   match(Set dst (OrI src1 src2));
14797 
14798   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14799 
14800   ins_cost(INSN_COST);
14801   ins_encode %{
14802     __ orrw(as_Register($dst$$reg),
14803             as_Register($src1$$reg),
14804             (uint64_t)($src2$$constant));
14805   %}
14806 
14807   ins_pipe(ialu_reg_imm);
14808 %}
14809 
14810 // Xor Instructions
14811 
14812 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14813   match(Set dst (XorI src1 src2));
14814 
14815   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14816 
14817   ins_cost(INSN_COST);
14818   ins_encode %{
14819     __ eorw(as_Register($dst$$reg),
14820             as_Register($src1$$reg),
14821             as_Register($src2$$reg));
14822   %}
14823 
14824   ins_pipe(ialu_reg_reg);
14825 %}
14826 
14827 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14828   match(Set dst (XorI src1 src2));
14829 
14830   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14831 
14832   ins_cost(INSN_COST);
14833   ins_encode %{
14834     __ eorw(as_Register($dst$$reg),
14835             as_Register($src1$$reg),
14836             (uint64_t)($src2$$constant));
14837   %}
14838 
14839   ins_pipe(ialu_reg_imm);
14840 %}
14841 
14842 // Long Logical Instructions
14843 // TODO
14844 
14845 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14846   match(Set dst (AndL src1 src2));
14847 
14848   format %{ "and  $dst, $src1, $src2\t# int" %}
14849 
14850   ins_cost(INSN_COST);
14851   ins_encode %{
14852     __ andr(as_Register($dst$$reg),
14853             as_Register($src1$$reg),
14854             as_Register($src2$$reg));
14855   %}
14856 
14857   ins_pipe(ialu_reg_reg);
14858 %}
14859 
14860 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14861   match(Set dst (AndL src1 src2));
14862 
14863   format %{ "and  $dst, $src1, $src2\t# int" %}
14864 
14865   ins_cost(INSN_COST);
14866   ins_encode %{
14867     __ andr(as_Register($dst$$reg),
14868             as_Register($src1$$reg),
14869             (uint64_t)($src2$$constant));
14870   %}
14871 
14872   ins_pipe(ialu_reg_imm);
14873 %}
14874 
14875 // Or Instructions
14876 
14877 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14878   match(Set dst (OrL src1 src2));
14879 
14880   format %{ "orr  $dst, $src1, $src2\t# int" %}
14881 
14882   ins_cost(INSN_COST);
14883   ins_encode %{
14884     __ orr(as_Register($dst$$reg),
14885            as_Register($src1$$reg),
14886            as_Register($src2$$reg));
14887   %}
14888 
14889   ins_pipe(ialu_reg_reg);
14890 %}
14891 
14892 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14893   match(Set dst (OrL src1 src2));
14894 
14895   format %{ "orr  $dst, $src1, $src2\t# int" %}
14896 
14897   ins_cost(INSN_COST);
14898   ins_encode %{
14899     __ orr(as_Register($dst$$reg),
14900            as_Register($src1$$reg),
14901            (uint64_t)($src2$$constant));
14902   %}
14903 
14904   ins_pipe(ialu_reg_imm);
14905 %}
14906 
14907 // Xor Instructions
14908 
14909 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14910   match(Set dst (XorL src1 src2));
14911 
14912   format %{ "eor  $dst, $src1, $src2\t# int" %}
14913 
14914   ins_cost(INSN_COST);
14915   ins_encode %{
14916     __ eor(as_Register($dst$$reg),
14917            as_Register($src1$$reg),
14918            as_Register($src2$$reg));
14919   %}
14920 
14921   ins_pipe(ialu_reg_reg);
14922 %}
14923 
14924 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14925   match(Set dst (XorL src1 src2));
14926 
14927   ins_cost(INSN_COST);
14928   format %{ "eor  $dst, $src1, $src2\t# int" %}
14929 
14930   ins_encode %{
14931     __ eor(as_Register($dst$$reg),
14932            as_Register($src1$$reg),
14933            (uint64_t)($src2$$constant));
14934   %}
14935 
14936   ins_pipe(ialu_reg_imm);
14937 %}
14938 
14939 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14940 %{
14941   match(Set dst (ConvI2L src));
14942 
14943   ins_cost(INSN_COST);
14944   format %{ "sxtw  $dst, $src\t# i2l" %}
14945   ins_encode %{
14946     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14947   %}
14948   ins_pipe(ialu_reg_shift);
14949 %}
14950 
14951 // this pattern occurs in bigmath arithmetic
14952 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14953 %{
14954   match(Set dst (AndL (ConvI2L src) mask));
14955 
14956   ins_cost(INSN_COST);
14957   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14958   ins_encode %{
14959     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14960   %}
14961 
14962   ins_pipe(ialu_reg_shift);
14963 %}
14964 
14965 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14966   match(Set dst (ConvL2I src));
14967 
14968   ins_cost(INSN_COST);
14969   format %{ "movw  $dst, $src \t// l2i" %}
14970 
14971   ins_encode %{
14972     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14973   %}
14974 
14975   ins_pipe(ialu_reg);
14976 %}
14977 
14978 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14979 %{
14980   match(Set dst (Conv2B src));
14981   effect(KILL cr);
14982 
14983   format %{
14984     "cmpw $src, zr\n\t"
14985     "cset $dst, ne"
14986   %}
14987 
14988   ins_encode %{
14989     __ cmpw(as_Register($src$$reg), zr);
14990     __ cset(as_Register($dst$$reg), Assembler::NE);
14991   %}
14992 
14993   ins_pipe(ialu_reg);
14994 %}
14995 
14996 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14997 %{
14998   match(Set dst (Conv2B src));
14999   effect(KILL cr);
15000 
15001   format %{
15002     "cmp  $src, zr\n\t"
15003     "cset $dst, ne"
15004   %}
15005 
15006   ins_encode %{
15007     __ cmp(as_Register($src$$reg), zr);
15008     __ cset(as_Register($dst$$reg), Assembler::NE);
15009   %}
15010 
15011   ins_pipe(ialu_reg);
15012 %}
15013 
15014 instruct convD2F_reg(vRegF dst, vRegD src) %{
15015   match(Set dst (ConvD2F src));
15016 
15017   ins_cost(INSN_COST * 5);
15018   format %{ "fcvtd  $dst, $src \t// d2f" %}
15019 
15020   ins_encode %{
15021     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15022   %}
15023 
15024   ins_pipe(fp_d2f);
15025 %}
15026 
15027 instruct convF2D_reg(vRegD dst, vRegF src) %{
15028   match(Set dst (ConvF2D src));
15029 
15030   ins_cost(INSN_COST * 5);
15031   format %{ "fcvts  $dst, $src \t// f2d" %}
15032 
15033   ins_encode %{
15034     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15035   %}
15036 
15037   ins_pipe(fp_f2d);
15038 %}
15039 
15040 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15041   match(Set dst (ConvF2I src));
15042 
15043   ins_cost(INSN_COST * 5);
15044   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
15045 
15046   ins_encode %{
15047     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15048   %}
15049 
15050   ins_pipe(fp_f2i);
15051 %}
15052 
15053 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15054   match(Set dst (ConvF2L src));
15055 
15056   ins_cost(INSN_COST * 5);
15057   format %{ "fcvtzs  $dst, $src \t// f2l" %}
15058 
15059   ins_encode %{
15060     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15061   %}
15062 
15063   ins_pipe(fp_f2l);
15064 %}
15065 
15066 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15067   match(Set dst (ConvI2F src));
15068 
15069   ins_cost(INSN_COST * 5);
15070   format %{ "scvtfws  $dst, $src \t// i2f" %}
15071 
15072   ins_encode %{
15073     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15074   %}
15075 
15076   ins_pipe(fp_i2f);
15077 %}
15078 
15079 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15080   match(Set dst (ConvL2F src));
15081 
15082   ins_cost(INSN_COST * 5);
15083   format %{ "scvtfs  $dst, $src \t// l2f" %}
15084 
15085   ins_encode %{
15086     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15087   %}
15088 
15089   ins_pipe(fp_l2f);
15090 %}
15091 
15092 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15093   match(Set dst (ConvD2I src));
15094 
15095   ins_cost(INSN_COST * 5);
15096   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15097 
15098   ins_encode %{
15099     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15100   %}
15101 
15102   ins_pipe(fp_d2i);
15103 %}
15104 
15105 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15106   match(Set dst (ConvD2L src));
15107 
15108   ins_cost(INSN_COST * 5);
15109   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15110 
15111   ins_encode %{
15112     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15113   %}
15114 
15115   ins_pipe(fp_d2l);
15116 %}
15117 
15118 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15119   match(Set dst (ConvI2D src));
15120 
15121   ins_cost(INSN_COST * 5);
15122   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15123 
15124   ins_encode %{
15125     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15126   %}
15127 
15128   ins_pipe(fp_i2d);
15129 %}
15130 
15131 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15132   match(Set dst (ConvL2D src));
15133 
15134   ins_cost(INSN_COST * 5);
15135   format %{ "scvtfd  $dst, $src \t// l2d" %}
15136 
15137   ins_encode %{
15138     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15139   %}
15140 
15141   ins_pipe(fp_l2d);
15142 %}
15143 
15144 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15145 %{
15146   match(Set dst (RoundD src));
15147   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15148   format %{ "java_round_double $dst,$src"%}
15149   ins_encode %{
15150     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15151                          as_FloatRegister($ftmp$$reg));
15152   %}
15153   ins_pipe(pipe_slow);
15154 %}
15155 
15156 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15157 %{
15158   match(Set dst (RoundF src));
15159   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15160   format %{ "java_round_float $dst,$src"%}
15161   ins_encode %{
15162     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15163                         as_FloatRegister($ftmp$$reg));
15164   %}
15165   ins_pipe(pipe_slow);
15166 %}
15167 
15168 // stack <-> reg and reg <-> reg shuffles with no conversion
15169 
15170 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15171 
15172   match(Set dst (MoveF2I src));
15173 
15174   effect(DEF dst, USE src);
15175 
15176   ins_cost(4 * INSN_COST);
15177 
15178   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15179 
15180   ins_encode %{
15181     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15182   %}
15183 
15184   ins_pipe(iload_reg_reg);
15185 
15186 %}
15187 
15188 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15189 
15190   match(Set dst (MoveI2F src));
15191 
15192   effect(DEF dst, USE src);
15193 
15194   ins_cost(4 * INSN_COST);
15195 
15196   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15197 
15198   ins_encode %{
15199     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15200   %}
15201 
15202   ins_pipe(pipe_class_memory);
15203 
15204 %}
15205 
15206 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15207 
15208   match(Set dst (MoveD2L src));
15209 
15210   effect(DEF dst, USE src);
15211 
15212   ins_cost(4 * INSN_COST);
15213 
15214   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15215 
15216   ins_encode %{
15217     __ ldr($dst$$Register, Address(sp, $src$$disp));
15218   %}
15219 
15220   ins_pipe(iload_reg_reg);
15221 
15222 %}
15223 
15224 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15225 
15226   match(Set dst (MoveL2D src));
15227 
15228   effect(DEF dst, USE src);
15229 
15230   ins_cost(4 * INSN_COST);
15231 
15232   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15233 
15234   ins_encode %{
15235     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15236   %}
15237 
15238   ins_pipe(pipe_class_memory);
15239 
15240 %}
15241 
15242 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15243 
15244   match(Set dst (MoveF2I src));
15245 
15246   effect(DEF dst, USE src);
15247 
15248   ins_cost(INSN_COST);
15249 
15250   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15251 
15252   ins_encode %{
15253     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15254   %}
15255 
15256   ins_pipe(pipe_class_memory);
15257 
15258 %}
15259 
15260 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15261 
15262   match(Set dst (MoveI2F src));
15263 
15264   effect(DEF dst, USE src);
15265 
15266   ins_cost(INSN_COST);
15267 
15268   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15269 
15270   ins_encode %{
15271     __ strw($src$$Register, Address(sp, $dst$$disp));
15272   %}
15273 
15274   ins_pipe(istore_reg_reg);
15275 
15276 %}
15277 
15278 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15279 
15280   match(Set dst (MoveD2L src));
15281 
15282   effect(DEF dst, USE src);
15283 
15284   ins_cost(INSN_COST);
15285 
15286   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15287 
15288   ins_encode %{
15289     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15290   %}
15291 
15292   ins_pipe(pipe_class_memory);
15293 
15294 %}
15295 
15296 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15297 
15298   match(Set dst (MoveL2D src));
15299 
15300   effect(DEF dst, USE src);
15301 
15302   ins_cost(INSN_COST);
15303 
15304   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15305 
15306   ins_encode %{
15307     __ str($src$$Register, Address(sp, $dst$$disp));
15308   %}
15309 
15310   ins_pipe(istore_reg_reg);
15311 
15312 %}
15313 
15314 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15315 
15316   match(Set dst (MoveF2I src));
15317 
15318   effect(DEF dst, USE src);
15319 
15320   ins_cost(INSN_COST);
15321 
15322   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15323 
15324   ins_encode %{
15325     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15326   %}
15327 
15328   ins_pipe(fp_f2i);
15329 
15330 %}
15331 
15332 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15333 
15334   match(Set dst (MoveI2F src));
15335 
15336   effect(DEF dst, USE src);
15337 
15338   ins_cost(INSN_COST);
15339 
15340   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15341 
15342   ins_encode %{
15343     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15344   %}
15345 
15346   ins_pipe(fp_i2f);
15347 
15348 %}
15349 
15350 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15351 
15352   match(Set dst (MoveD2L src));
15353 
15354   effect(DEF dst, USE src);
15355 
15356   ins_cost(INSN_COST);
15357 
15358   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15359 
15360   ins_encode %{
15361     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15362   %}
15363 
15364   ins_pipe(fp_d2l);
15365 
15366 %}
15367 
15368 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15369 
15370   match(Set dst (MoveL2D src));
15371 
15372   effect(DEF dst, USE src);
15373 
15374   ins_cost(INSN_COST);
15375 
15376   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15377 
15378   ins_encode %{
15379     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15380   %}
15381 
15382   ins_pipe(fp_l2d);
15383 
15384 %}
15385 
15386 // ============================================================================
15387 // clearing of an array
15388 
15389 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15390 %{
15391   match(Set dummy (ClearArray cnt base));
15392   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15393 
15394   ins_cost(4 * INSN_COST);
15395   format %{ "ClearArray $cnt, $base" %}
15396 
15397   ins_encode %{
15398     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15399     if (tpc == NULL) {
15400       ciEnv::current()->record_failure("CodeCache is full");
15401       return;
15402     }
15403   %}
15404 
15405   ins_pipe(pipe_class_memory);
15406 %}
15407 
15408 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15409 %{
15410   predicate((uint64_t)n->in(2)->get_long()
15411             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15412   match(Set dummy (ClearArray cnt base));
15413   effect(TEMP temp, USE_KILL base, KILL cr);
15414 
15415   ins_cost(4 * INSN_COST);
15416   format %{ "ClearArray $cnt, $base" %}
15417 
15418   ins_encode %{
15419     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15420   %}
15421 
15422   ins_pipe(pipe_class_memory);
15423 %}
15424 
15425 // ============================================================================
15426 // Overflow Math Instructions
15427 
15428 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15429 %{
15430   match(Set cr (OverflowAddI op1 op2));
15431 
15432   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15433   ins_cost(INSN_COST);
15434   ins_encode %{
15435     __ cmnw($op1$$Register, $op2$$Register);
15436   %}
15437 
15438   ins_pipe(icmp_reg_reg);
15439 %}
15440 
15441 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15442 %{
15443   match(Set cr (OverflowAddI op1 op2));
15444 
15445   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15446   ins_cost(INSN_COST);
15447   ins_encode %{
15448     __ cmnw($op1$$Register, $op2$$constant);
15449   %}
15450 
15451   ins_pipe(icmp_reg_imm);
15452 %}
15453 
15454 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15455 %{
15456   match(Set cr (OverflowAddL op1 op2));
15457 
15458   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15459   ins_cost(INSN_COST);
15460   ins_encode %{
15461     __ cmn($op1$$Register, $op2$$Register);
15462   %}
15463 
15464   ins_pipe(icmp_reg_reg);
15465 %}
15466 
15467 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15468 %{
15469   match(Set cr (OverflowAddL op1 op2));
15470 
15471   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15472   ins_cost(INSN_COST);
15473   ins_encode %{
15474     __ cmn($op1$$Register, $op2$$constant);
15475   %}
15476 
15477   ins_pipe(icmp_reg_imm);
15478 %}
15479 
15480 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15481 %{
15482   match(Set cr (OverflowSubI op1 op2));
15483 
15484   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15485   ins_cost(INSN_COST);
15486   ins_encode %{
15487     __ cmpw($op1$$Register, $op2$$Register);
15488   %}
15489 
15490   ins_pipe(icmp_reg_reg);
15491 %}
15492 
15493 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15494 %{
15495   match(Set cr (OverflowSubI op1 op2));
15496 
15497   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15498   ins_cost(INSN_COST);
15499   ins_encode %{
15500     __ cmpw($op1$$Register, $op2$$constant);
15501   %}
15502 
15503   ins_pipe(icmp_reg_imm);
15504 %}
15505 
15506 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15507 %{
15508   match(Set cr (OverflowSubL op1 op2));
15509 
15510   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15511   ins_cost(INSN_COST);
15512   ins_encode %{
15513     __ cmp($op1$$Register, $op2$$Register);
15514   %}
15515 
15516   ins_pipe(icmp_reg_reg);
15517 %}
15518 
15519 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15520 %{
15521   match(Set cr (OverflowSubL op1 op2));
15522 
15523   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15524   ins_cost(INSN_COST);
15525   ins_encode %{
15526     __ subs(zr, $op1$$Register, $op2$$constant);
15527   %}
15528 
15529   ins_pipe(icmp_reg_imm);
15530 %}
15531 
15532 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15533 %{
15534   match(Set cr (OverflowSubI zero op1));
15535 
15536   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15537   ins_cost(INSN_COST);
15538   ins_encode %{
15539     __ cmpw(zr, $op1$$Register);
15540   %}
15541 
15542   ins_pipe(icmp_reg_imm);
15543 %}
15544 
15545 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15546 %{
15547   match(Set cr (OverflowSubL zero op1));
15548 
15549   format %{ "cmp   zr, $op1\t# overflow check long" %}
15550   ins_cost(INSN_COST);
15551   ins_encode %{
15552     __ cmp(zr, $op1$$Register);
15553   %}
15554 
15555   ins_pipe(icmp_reg_imm);
15556 %}
15557 
15558 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15559 %{
15560   match(Set cr (OverflowMulI op1 op2));
15561 
15562   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15563             "cmp   rscratch1, rscratch1, sxtw\n\t"
15564             "movw  rscratch1, #0x80000000\n\t"
15565             "cselw rscratch1, rscratch1, zr, NE\n\t"
15566             "cmpw  rscratch1, #1" %}
15567   ins_cost(5 * INSN_COST);
15568   ins_encode %{
15569     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15570     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15571     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15572     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15573     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15574   %}
15575 
15576   ins_pipe(pipe_slow);
15577 %}
15578 
15579 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15580 %{
15581   match(If cmp (OverflowMulI op1 op2));
15582   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15583             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15584   effect(USE labl, KILL cr);
15585 
15586   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15587             "cmp   rscratch1, rscratch1, sxtw\n\t"
15588             "b$cmp   $labl" %}
15589   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15590   ins_encode %{
15591     Label* L = $labl$$label;
15592     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15593     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15594     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15595     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15596   %}
15597 
15598   ins_pipe(pipe_serial);
15599 %}
15600 
15601 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15602 %{
15603   match(Set cr (OverflowMulL op1 op2));
15604 
15605   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15606             "smulh rscratch2, $op1, $op2\n\t"
15607             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15608             "movw  rscratch1, #0x80000000\n\t"
15609             "cselw rscratch1, rscratch1, zr, NE\n\t"
15610             "cmpw  rscratch1, #1" %}
15611   ins_cost(6 * INSN_COST);
15612   ins_encode %{
15613     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15614     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15615     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15616     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15617     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15618     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15619   %}
15620 
15621   ins_pipe(pipe_slow);
15622 %}
15623 
15624 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15625 %{
15626   match(If cmp (OverflowMulL op1 op2));
15627   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15628             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15629   effect(USE labl, KILL cr);
15630 
15631   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15632             "smulh rscratch2, $op1, $op2\n\t"
15633             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15634             "b$cmp $labl" %}
15635   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15636   ins_encode %{
15637     Label* L = $labl$$label;
15638     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15639     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15640     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15641     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15642     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15643   %}
15644 
15645   ins_pipe(pipe_serial);
15646 %}
15647 
15648 // ============================================================================
15649 // Compare Instructions
15650 
15651 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15652 %{
15653   match(Set cr (CmpI op1 op2));
15654 
15655   effect(DEF cr, USE op1, USE op2);
15656 
15657   ins_cost(INSN_COST);
15658   format %{ "cmpw  $op1, $op2" %}
15659 
15660   ins_encode(aarch64_enc_cmpw(op1, op2));
15661 
15662   ins_pipe(icmp_reg_reg);
15663 %}
15664 
15665 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15666 %{
15667   match(Set cr (CmpI op1 zero));
15668 
15669   effect(DEF cr, USE op1);
15670 
15671   ins_cost(INSN_COST);
15672   format %{ "cmpw $op1, 0" %}
15673 
15674   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15675 
15676   ins_pipe(icmp_reg_imm);
15677 %}
15678 
15679 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15680 %{
15681   match(Set cr (CmpI op1 op2));
15682 
15683   effect(DEF cr, USE op1);
15684 
15685   ins_cost(INSN_COST);
15686   format %{ "cmpw  $op1, $op2" %}
15687 
15688   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15689 
15690   ins_pipe(icmp_reg_imm);
15691 %}
15692 
15693 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15694 %{
15695   match(Set cr (CmpI op1 op2));
15696 
15697   effect(DEF cr, USE op1);
15698 
15699   ins_cost(INSN_COST * 2);
15700   format %{ "cmpw  $op1, $op2" %}
15701 
15702   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15703 
15704   ins_pipe(icmp_reg_imm);
15705 %}
15706 
15707 // Unsigned compare Instructions; really, same as signed compare
15708 // except it should only be used to feed an If or a CMovI which takes a
15709 // cmpOpU.
15710 
15711 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15712 %{
15713   match(Set cr (CmpU op1 op2));
15714 
15715   effect(DEF cr, USE op1, USE op2);
15716 
15717   ins_cost(INSN_COST);
15718   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15719 
15720   ins_encode(aarch64_enc_cmpw(op1, op2));
15721 
15722   ins_pipe(icmp_reg_reg);
15723 %}
15724 
15725 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15726 %{
15727   match(Set cr (CmpU op1 zero));
15728 
15729   effect(DEF cr, USE op1);
15730 
15731   ins_cost(INSN_COST);
15732   format %{ "cmpw $op1, #0\t# unsigned" %}
15733 
15734   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15735 
15736   ins_pipe(icmp_reg_imm);
15737 %}
15738 
15739 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15740 %{
15741   match(Set cr (CmpU op1 op2));
15742 
15743   effect(DEF cr, USE op1);
15744 
15745   ins_cost(INSN_COST);
15746   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15747 
15748   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15749 
15750   ins_pipe(icmp_reg_imm);
15751 %}
15752 
15753 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15754 %{
15755   match(Set cr (CmpU op1 op2));
15756 
15757   effect(DEF cr, USE op1);
15758 
15759   ins_cost(INSN_COST * 2);
15760   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15761 
15762   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15763 
15764   ins_pipe(icmp_reg_imm);
15765 %}
15766 
15767 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15768 %{
15769   match(Set cr (CmpL op1 op2));
15770 
15771   effect(DEF cr, USE op1, USE op2);
15772 
15773   ins_cost(INSN_COST);
15774   format %{ "cmp  $op1, $op2" %}
15775 
15776   ins_encode(aarch64_enc_cmp(op1, op2));
15777 
15778   ins_pipe(icmp_reg_reg);
15779 %}
15780 
15781 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15782 %{
15783   match(Set cr (CmpL op1 zero));
15784 
15785   effect(DEF cr, USE op1);
15786 
15787   ins_cost(INSN_COST);
15788   format %{ "tst  $op1" %}
15789 
15790   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15791 
15792   ins_pipe(icmp_reg_imm);
15793 %}
15794 
15795 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15796 %{
15797   match(Set cr (CmpL op1 op2));
15798 
15799   effect(DEF cr, USE op1);
15800 
15801   ins_cost(INSN_COST);
15802   format %{ "cmp  $op1, $op2" %}
15803 
15804   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15805 
15806   ins_pipe(icmp_reg_imm);
15807 %}
15808 
15809 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15810 %{
15811   match(Set cr (CmpL op1 op2));
15812 
15813   effect(DEF cr, USE op1);
15814 
15815   ins_cost(INSN_COST * 2);
15816   format %{ "cmp  $op1, $op2" %}
15817 
15818   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15819 
15820   ins_pipe(icmp_reg_imm);
15821 %}
15822 
15823 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15824 %{
15825   match(Set cr (CmpUL op1 op2));
15826 
15827   effect(DEF cr, USE op1, USE op2);
15828 
15829   ins_cost(INSN_COST);
15830   format %{ "cmp  $op1, $op2" %}
15831 
15832   ins_encode(aarch64_enc_cmp(op1, op2));
15833 
15834   ins_pipe(icmp_reg_reg);
15835 %}
15836 
15837 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15838 %{
15839   match(Set cr (CmpUL op1 zero));
15840 
15841   effect(DEF cr, USE op1);
15842 
15843   ins_cost(INSN_COST);
15844   format %{ "tst  $op1" %}
15845 
15846   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15847 
15848   ins_pipe(icmp_reg_imm);
15849 %}
15850 
15851 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15852 %{
15853   match(Set cr (CmpUL op1 op2));
15854 
15855   effect(DEF cr, USE op1);
15856 
15857   ins_cost(INSN_COST);
15858   format %{ "cmp  $op1, $op2" %}
15859 
15860   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15861 
15862   ins_pipe(icmp_reg_imm);
15863 %}
15864 
15865 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15866 %{
15867   match(Set cr (CmpUL op1 op2));
15868 
15869   effect(DEF cr, USE op1);
15870 
15871   ins_cost(INSN_COST * 2);
15872   format %{ "cmp  $op1, $op2" %}
15873 
15874   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15875 
15876   ins_pipe(icmp_reg_imm);
15877 %}
15878 
15879 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15880 %{
15881   match(Set cr (CmpP op1 op2));
15882 
15883   effect(DEF cr, USE op1, USE op2);
15884 
15885   ins_cost(INSN_COST);
15886   format %{ "cmp  $op1, $op2\t // ptr" %}
15887 
15888   ins_encode(aarch64_enc_cmpp(op1, op2));
15889 
15890   ins_pipe(icmp_reg_reg);
15891 %}
15892 
15893 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15894 %{
15895   match(Set cr (CmpN op1 op2));
15896 
15897   effect(DEF cr, USE op1, USE op2);
15898 
15899   ins_cost(INSN_COST);
15900   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15901 
15902   ins_encode(aarch64_enc_cmpn(op1, op2));
15903 
15904   ins_pipe(icmp_reg_reg);
15905 %}
15906 
15907 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15908 %{
15909   match(Set cr (CmpP op1 zero));
15910 
15911   effect(DEF cr, USE op1, USE zero);
15912 
15913   ins_cost(INSN_COST);
15914   format %{ "cmp  $op1, 0\t // ptr" %}
15915 
15916   ins_encode(aarch64_enc_testp(op1));
15917 
15918   ins_pipe(icmp_reg_imm);
15919 %}
15920 
15921 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15922 %{
15923   match(Set cr (CmpN op1 zero));
15924 
15925   effect(DEF cr, USE op1, USE zero);
15926 
15927   ins_cost(INSN_COST);
15928   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15929 
15930   ins_encode(aarch64_enc_testn(op1));
15931 
15932   ins_pipe(icmp_reg_imm);
15933 %}
15934 
15935 // FP comparisons
15936 //
15937 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15938 // using normal cmpOp. See declaration of rFlagsReg for details.
15939 
15940 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15941 %{
15942   match(Set cr (CmpF src1 src2));
15943 
15944   ins_cost(3 * INSN_COST);
15945   format %{ "fcmps $src1, $src2" %}
15946 
15947   ins_encode %{
15948     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15949   %}
15950 
15951   ins_pipe(pipe_class_compare);
15952 %}
15953 
15954 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15955 %{
15956   match(Set cr (CmpF src1 src2));
15957 
15958   ins_cost(3 * INSN_COST);
15959   format %{ "fcmps $src1, 0.0" %}
15960 
15961   ins_encode %{
15962     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15963   %}
15964 
15965   ins_pipe(pipe_class_compare);
15966 %}
15967 // FROM HERE
15968 
15969 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15970 %{
15971   match(Set cr (CmpD src1 src2));
15972 
15973   ins_cost(3 * INSN_COST);
15974   format %{ "fcmpd $src1, $src2" %}
15975 
15976   ins_encode %{
15977     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15978   %}
15979 
15980   ins_pipe(pipe_class_compare);
15981 %}
15982 
15983 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15984 %{
15985   match(Set cr (CmpD src1 src2));
15986 
15987   ins_cost(3 * INSN_COST);
15988   format %{ "fcmpd $src1, 0.0" %}
15989 
15990   ins_encode %{
15991     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15992   %}
15993 
15994   ins_pipe(pipe_class_compare);
15995 %}
15996 
15997 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15998 %{
15999   match(Set dst (CmpF3 src1 src2));
16000   effect(KILL cr);
16001 
16002   ins_cost(5 * INSN_COST);
16003   format %{ "fcmps $src1, $src2\n\t"
16004             "csinvw($dst, zr, zr, eq\n\t"
16005             "csnegw($dst, $dst, $dst, lt)"
16006   %}
16007 
16008   ins_encode %{
16009     Label done;
16010     FloatRegister s1 = as_FloatRegister($src1$$reg);
16011     FloatRegister s2 = as_FloatRegister($src2$$reg);
16012     Register d = as_Register($dst$$reg);
16013     __ fcmps(s1, s2);
16014     // installs 0 if EQ else -1
16015     __ csinvw(d, zr, zr, Assembler::EQ);
16016     // keeps -1 if less or unordered else installs 1
16017     __ csnegw(d, d, d, Assembler::LT);
16018     __ bind(done);
16019   %}
16020 
16021   ins_pipe(pipe_class_default);
16022 
16023 %}
16024 
16025 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16026 %{
16027   match(Set dst (CmpD3 src1 src2));
16028   effect(KILL cr);
16029 
16030   ins_cost(5 * INSN_COST);
16031   format %{ "fcmpd $src1, $src2\n\t"
16032             "csinvw($dst, zr, zr, eq\n\t"
16033             "csnegw($dst, $dst, $dst, lt)"
16034   %}
16035 
16036   ins_encode %{
16037     Label done;
16038     FloatRegister s1 = as_FloatRegister($src1$$reg);
16039     FloatRegister s2 = as_FloatRegister($src2$$reg);
16040     Register d = as_Register($dst$$reg);
16041     __ fcmpd(s1, s2);
16042     // installs 0 if EQ else -1
16043     __ csinvw(d, zr, zr, Assembler::EQ);
16044     // keeps -1 if less or unordered else installs 1
16045     __ csnegw(d, d, d, Assembler::LT);
16046     __ bind(done);
16047   %}
16048   ins_pipe(pipe_class_default);
16049 
16050 %}
16051 
16052 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16053 %{
16054   match(Set dst (CmpF3 src1 zero));
16055   effect(KILL cr);
16056 
16057   ins_cost(5 * INSN_COST);
16058   format %{ "fcmps $src1, 0.0\n\t"
16059             "csinvw($dst, zr, zr, eq\n\t"
16060             "csnegw($dst, $dst, $dst, lt)"
16061   %}
16062 
16063   ins_encode %{
16064     Label done;
16065     FloatRegister s1 = as_FloatRegister($src1$$reg);
16066     Register d = as_Register($dst$$reg);
16067     __ fcmps(s1, 0.0);
16068     // installs 0 if EQ else -1
16069     __ csinvw(d, zr, zr, Assembler::EQ);
16070     // keeps -1 if less or unordered else installs 1
16071     __ csnegw(d, d, d, Assembler::LT);
16072     __ bind(done);
16073   %}
16074 
16075   ins_pipe(pipe_class_default);
16076 
16077 %}
16078 
16079 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16080 %{
16081   match(Set dst (CmpD3 src1 zero));
16082   effect(KILL cr);
16083 
16084   ins_cost(5 * INSN_COST);
16085   format %{ "fcmpd $src1, 0.0\n\t"
16086             "csinvw($dst, zr, zr, eq\n\t"
16087             "csnegw($dst, $dst, $dst, lt)"
16088   %}
16089 
16090   ins_encode %{
16091     Label done;
16092     FloatRegister s1 = as_FloatRegister($src1$$reg);
16093     Register d = as_Register($dst$$reg);
16094     __ fcmpd(s1, 0.0);
16095     // installs 0 if EQ else -1
16096     __ csinvw(d, zr, zr, Assembler::EQ);
16097     // keeps -1 if less or unordered else installs 1
16098     __ csnegw(d, d, d, Assembler::LT);
16099     __ bind(done);
16100   %}
16101   ins_pipe(pipe_class_default);
16102 
16103 %}
16104 
16105 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16106 %{
16107   match(Set dst (CmpLTMask p q));
16108   effect(KILL cr);
16109 
16110   ins_cost(3 * INSN_COST);
16111 
16112   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16113             "csetw $dst, lt\n\t"
16114             "subw $dst, zr, $dst"
16115   %}
16116 
16117   ins_encode %{
16118     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16119     __ csetw(as_Register($dst$$reg), Assembler::LT);
16120     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16121   %}
16122 
16123   ins_pipe(ialu_reg_reg);
16124 %}
16125 
16126 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16127 %{
16128   match(Set dst (CmpLTMask src zero));
16129   effect(KILL cr);
16130 
16131   ins_cost(INSN_COST);
16132 
16133   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16134 
16135   ins_encode %{
16136     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16137   %}
16138 
16139   ins_pipe(ialu_reg_shift);
16140 %}
16141 
16142 // ============================================================================
16143 // Max and Min
16144 
16145 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16146 %{
16147   effect( DEF dst, USE src1, USE src2, USE cr );
16148 
16149   ins_cost(INSN_COST * 2);
16150   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
16151 
16152   ins_encode %{
16153     __ cselw(as_Register($dst$$reg),
16154              as_Register($src1$$reg),
16155              as_Register($src2$$reg),
16156              Assembler::LT);
16157   %}
16158 
16159   ins_pipe(icond_reg_reg);
16160 %}
16161 
16162 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16163 %{
16164   match(Set dst (MinI src1 src2));
16165   ins_cost(INSN_COST * 3);
16166 
16167   expand %{
16168     rFlagsReg cr;
16169     compI_reg_reg(cr, src1, src2);
16170     cmovI_reg_reg_lt(dst, src1, src2, cr);
16171   %}
16172 
16173 %}
16174 // FROM HERE
16175 
16176 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16177 %{
16178   effect( DEF dst, USE src1, USE src2, USE cr );
16179 
16180   ins_cost(INSN_COST * 2);
16181   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
16182 
16183   ins_encode %{
16184     __ cselw(as_Register($dst$$reg),
16185              as_Register($src1$$reg),
16186              as_Register($src2$$reg),
16187              Assembler::GT);
16188   %}
16189 
16190   ins_pipe(icond_reg_reg);
16191 %}
16192 
16193 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16194 %{
16195   match(Set dst (MaxI src1 src2));
16196   ins_cost(INSN_COST * 3);
16197   expand %{
16198     rFlagsReg cr;
16199     compI_reg_reg(cr, src1, src2);
16200     cmovI_reg_reg_gt(dst, src1, src2, cr);
16201   %}
16202 %}
16203 
16204 // ============================================================================
16205 // Branch Instructions
16206 
16207 // Direct Branch.
16208 instruct branch(label lbl)
16209 %{
16210   match(Goto);
16211 
16212   effect(USE lbl);
16213 
16214   ins_cost(BRANCH_COST);
16215   format %{ "b  $lbl" %}
16216 
16217   ins_encode(aarch64_enc_b(lbl));
16218 
16219   ins_pipe(pipe_branch);
16220 %}
16221 
16222 // Conditional Near Branch
16223 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16224 %{
16225   // Same match rule as `branchConFar'.
16226   match(If cmp cr);
16227 
16228   effect(USE lbl);
16229 
16230   ins_cost(BRANCH_COST);
16231   // If set to 1 this indicates that the current instruction is a
16232   // short variant of a long branch. This avoids using this
16233   // instruction in first-pass matching. It will then only be used in
16234   // the `Shorten_branches' pass.
16235   // ins_short_branch(1);
16236   format %{ "b$cmp  $lbl" %}
16237 
16238   ins_encode(aarch64_enc_br_con(cmp, lbl));
16239 
16240   ins_pipe(pipe_branch_cond);
16241 %}
16242 
16243 // Conditional Near Branch Unsigned
16244 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16245 %{
16246   // Same match rule as `branchConFar'.
16247   match(If cmp cr);
16248 
16249   effect(USE lbl);
16250 
16251   ins_cost(BRANCH_COST);
16252   // If set to 1 this indicates that the current instruction is a
16253   // short variant of a long branch. This avoids using this
16254   // instruction in first-pass matching. It will then only be used in
16255   // the `Shorten_branches' pass.
16256   // ins_short_branch(1);
16257   format %{ "b$cmp  $lbl\t# unsigned" %}
16258 
16259   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16260 
16261   ins_pipe(pipe_branch_cond);
16262 %}
16263 
16264 // Make use of CBZ and CBNZ.  These instructions, as well as being
16265 // shorter than (cmp; branch), have the additional benefit of not
16266 // killing the flags.
16267 
16268 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16269   match(If cmp (CmpI op1 op2));
16270   effect(USE labl);
16271 
16272   ins_cost(BRANCH_COST);
16273   format %{ "cbw$cmp   $op1, $labl" %}
16274   ins_encode %{
16275     Label* L = $labl$$label;
16276     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16277     if (cond == Assembler::EQ)
16278       __ cbzw($op1$$Register, *L);
16279     else
16280       __ cbnzw($op1$$Register, *L);
16281   %}
16282   ins_pipe(pipe_cmp_branch);
16283 %}
16284 
16285 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16286   match(If cmp (CmpL op1 op2));
16287   effect(USE labl);
16288 
16289   ins_cost(BRANCH_COST);
16290   format %{ "cb$cmp   $op1, $labl" %}
16291   ins_encode %{
16292     Label* L = $labl$$label;
16293     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16294     if (cond == Assembler::EQ)
16295       __ cbz($op1$$Register, *L);
16296     else
16297       __ cbnz($op1$$Register, *L);
16298   %}
16299   ins_pipe(pipe_cmp_branch);
16300 %}
16301 
16302 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16303   match(If cmp (CmpP op1 op2));
16304   effect(USE labl);
16305 
16306   ins_cost(BRANCH_COST);
16307   format %{ "cb$cmp   $op1, $labl" %}
16308   ins_encode %{
16309     Label* L = $labl$$label;
16310     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16311     if (cond == Assembler::EQ)
16312       __ cbz($op1$$Register, *L);
16313     else
16314       __ cbnz($op1$$Register, *L);
16315   %}
16316   ins_pipe(pipe_cmp_branch);
16317 %}
16318 
16319 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16320   match(If cmp (CmpN op1 op2));
16321   effect(USE labl);
16322 
16323   ins_cost(BRANCH_COST);
16324   format %{ "cbw$cmp   $op1, $labl" %}
16325   ins_encode %{
16326     Label* L = $labl$$label;
16327     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16328     if (cond == Assembler::EQ)
16329       __ cbzw($op1$$Register, *L);
16330     else
16331       __ cbnzw($op1$$Register, *L);
16332   %}
16333   ins_pipe(pipe_cmp_branch);
16334 %}
16335 
16336 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16337   match(If cmp (CmpP (DecodeN oop) zero));
16338   effect(USE labl);
16339 
16340   ins_cost(BRANCH_COST);
16341   format %{ "cb$cmp   $oop, $labl" %}
16342   ins_encode %{
16343     Label* L = $labl$$label;
16344     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16345     if (cond == Assembler::EQ)
16346       __ cbzw($oop$$Register, *L);
16347     else
16348       __ cbnzw($oop$$Register, *L);
16349   %}
16350   ins_pipe(pipe_cmp_branch);
16351 %}
16352 
16353 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16354   match(If cmp (CmpU op1 op2));
16355   effect(USE labl);
16356 
16357   ins_cost(BRANCH_COST);
16358   format %{ "cbw$cmp   $op1, $labl" %}
16359   ins_encode %{
16360     Label* L = $labl$$label;
16361     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16362     if (cond == Assembler::EQ || cond == Assembler::LS)
16363       __ cbzw($op1$$Register, *L);
16364     else
16365       __ cbnzw($op1$$Register, *L);
16366   %}
16367   ins_pipe(pipe_cmp_branch);
16368 %}
16369 
16370 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16371   match(If cmp (CmpUL op1 op2));
16372   effect(USE labl);
16373 
16374   ins_cost(BRANCH_COST);
16375   format %{ "cb$cmp   $op1, $labl" %}
16376   ins_encode %{
16377     Label* L = $labl$$label;
16378     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16379     if (cond == Assembler::EQ || cond == Assembler::LS)
16380       __ cbz($op1$$Register, *L);
16381     else
16382       __ cbnz($op1$$Register, *L);
16383   %}
16384   ins_pipe(pipe_cmp_branch);
16385 %}
16386 
16387 // Test bit and Branch
16388 
16389 // Patterns for short (< 32KiB) variants
16390 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16391   match(If cmp (CmpL op1 op2));
16392   effect(USE labl);
16393 
16394   ins_cost(BRANCH_COST);
16395   format %{ "cb$cmp   $op1, $labl # long" %}
16396   ins_encode %{
16397     Label* L = $labl$$label;
16398     Assembler::Condition cond =
16399       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16400     __ tbr(cond, $op1$$Register, 63, *L);
16401   %}
16402   ins_pipe(pipe_cmp_branch);
16403   ins_short_branch(1);
16404 %}
16405 
16406 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16407   match(If cmp (CmpI op1 op2));
16408   effect(USE labl);
16409 
16410   ins_cost(BRANCH_COST);
16411   format %{ "cb$cmp   $op1, $labl # int" %}
16412   ins_encode %{
16413     Label* L = $labl$$label;
16414     Assembler::Condition cond =
16415       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16416     __ tbr(cond, $op1$$Register, 31, *L);
16417   %}
16418   ins_pipe(pipe_cmp_branch);
16419   ins_short_branch(1);
16420 %}
16421 
16422 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16423   match(If cmp (CmpL (AndL op1 op2) op3));
16424   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16425   effect(USE labl);
16426 
16427   ins_cost(BRANCH_COST);
16428   format %{ "tb$cmp   $op1, $op2, $labl" %}
16429   ins_encode %{
16430     Label* L = $labl$$label;
16431     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16432     int bit = exact_log2_long($op2$$constant);
16433     __ tbr(cond, $op1$$Register, bit, *L);
16434   %}
16435   ins_pipe(pipe_cmp_branch);
16436   ins_short_branch(1);
16437 %}
16438 
16439 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16440   match(If cmp (CmpI (AndI op1 op2) op3));
16441   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16442   effect(USE labl);
16443 
16444   ins_cost(BRANCH_COST);
16445   format %{ "tb$cmp   $op1, $op2, $labl" %}
16446   ins_encode %{
16447     Label* L = $labl$$label;
16448     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16449     int bit = exact_log2((juint)$op2$$constant);
16450     __ tbr(cond, $op1$$Register, bit, *L);
16451   %}
16452   ins_pipe(pipe_cmp_branch);
16453   ins_short_branch(1);
16454 %}
16455 
16456 // And far variants
16457 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16458   match(If cmp (CmpL op1 op2));
16459   effect(USE labl);
16460 
16461   ins_cost(BRANCH_COST);
16462   format %{ "cb$cmp   $op1, $labl # long" %}
16463   ins_encode %{
16464     Label* L = $labl$$label;
16465     Assembler::Condition cond =
16466       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16467     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16468   %}
16469   ins_pipe(pipe_cmp_branch);
16470 %}
16471 
16472 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16473   match(If cmp (CmpI op1 op2));
16474   effect(USE labl);
16475 
16476   ins_cost(BRANCH_COST);
16477   format %{ "cb$cmp   $op1, $labl # int" %}
16478   ins_encode %{
16479     Label* L = $labl$$label;
16480     Assembler::Condition cond =
16481       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16482     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16483   %}
16484   ins_pipe(pipe_cmp_branch);
16485 %}
16486 
16487 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16488   match(If cmp (CmpL (AndL op1 op2) op3));
16489   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16490   effect(USE labl);
16491 
16492   ins_cost(BRANCH_COST);
16493   format %{ "tb$cmp   $op1, $op2, $labl" %}
16494   ins_encode %{
16495     Label* L = $labl$$label;
16496     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16497     int bit = exact_log2_long($op2$$constant);
16498     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16499   %}
16500   ins_pipe(pipe_cmp_branch);
16501 %}
16502 
16503 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16504   match(If cmp (CmpI (AndI op1 op2) op3));
16505   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16506   effect(USE labl);
16507 
16508   ins_cost(BRANCH_COST);
16509   format %{ "tb$cmp   $op1, $op2, $labl" %}
16510   ins_encode %{
16511     Label* L = $labl$$label;
16512     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16513     int bit = exact_log2((juint)$op2$$constant);
16514     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16515   %}
16516   ins_pipe(pipe_cmp_branch);
16517 %}
16518 
16519 // Test bits
16520 
16521 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16522   match(Set cr (CmpL (AndL op1 op2) op3));
16523   predicate(Assembler::operand_valid_for_logical_immediate
16524             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16525 
16526   ins_cost(INSN_COST);
16527   format %{ "tst $op1, $op2 # long" %}
16528   ins_encode %{
16529     __ tst($op1$$Register, $op2$$constant);
16530   %}
16531   ins_pipe(ialu_reg_reg);
16532 %}
16533 
16534 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16535   match(Set cr (CmpI (AndI op1 op2) op3));
16536   predicate(Assembler::operand_valid_for_logical_immediate
16537             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16538 
16539   ins_cost(INSN_COST);
16540   format %{ "tst $op1, $op2 # int" %}
16541   ins_encode %{
16542     __ tstw($op1$$Register, $op2$$constant);
16543   %}
16544   ins_pipe(ialu_reg_reg);
16545 %}
16546 
16547 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16548   match(Set cr (CmpL (AndL op1 op2) op3));
16549 
16550   ins_cost(INSN_COST);
16551   format %{ "tst $op1, $op2 # long" %}
16552   ins_encode %{
16553     __ tst($op1$$Register, $op2$$Register);
16554   %}
16555   ins_pipe(ialu_reg_reg);
16556 %}
16557 
16558 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16559   match(Set cr (CmpI (AndI op1 op2) op3));
16560 
16561   ins_cost(INSN_COST);
16562   format %{ "tstw $op1, $op2 # int" %}
16563   ins_encode %{
16564     __ tstw($op1$$Register, $op2$$Register);
16565   %}
16566   ins_pipe(ialu_reg_reg);
16567 %}
16568 
16569 
16570 // Conditional Far Branch
16571 // Conditional Far Branch Unsigned
16572 // TODO: fixme
16573 
16574 // counted loop end branch near
16575 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16576 %{
16577   match(CountedLoopEnd cmp cr);
16578 
16579   effect(USE lbl);
16580 
16581   ins_cost(BRANCH_COST);
16582   // short variant.
16583   // ins_short_branch(1);
16584   format %{ "b$cmp $lbl \t// counted loop end" %}
16585 
16586   ins_encode(aarch64_enc_br_con(cmp, lbl));
16587 
16588   ins_pipe(pipe_branch);
16589 %}
16590 
16591 // counted loop end branch near Unsigned
16592 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16593 %{
16594   match(CountedLoopEnd cmp cr);
16595 
16596   effect(USE lbl);
16597 
16598   ins_cost(BRANCH_COST);
16599   // short variant.
16600   // ins_short_branch(1);
16601   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16602 
16603   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16604 
16605   ins_pipe(pipe_branch);
16606 %}
16607 
16608 // counted loop end branch far
16609 // counted loop end branch far unsigned
16610 // TODO: fixme
16611 
16612 // ============================================================================
16613 // inlined locking and unlocking
16614 
16615 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16616 %{
16617   match(Set cr (FastLock object box));
16618   effect(TEMP tmp, TEMP tmp2);
16619 
16620   // TODO
16621   // identify correct cost
16622   ins_cost(5 * INSN_COST);
16623   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16624 
16625   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16626 
16627   ins_pipe(pipe_serial);
16628 %}
16629 
16630 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16631 %{
16632   match(Set cr (FastUnlock object box));
16633   effect(TEMP tmp, TEMP tmp2);
16634 
16635   ins_cost(5 * INSN_COST);
16636   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16637 
16638   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16639 
16640   ins_pipe(pipe_serial);
16641 %}
16642 
16643 
16644 // ============================================================================
16645 // Safepoint Instructions
16646 
16647 // TODO
16648 // provide a near and far version of this code
16649 
16650 instruct safePoint(rFlagsReg cr, iRegP poll)
16651 %{
16652   match(SafePoint poll);
16653   effect(KILL cr);
16654 
16655   format %{
16656     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16657   %}
16658   ins_encode %{
16659     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16660   %}
16661   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16662 %}
16663 
16664 
16665 // ============================================================================
16666 // Procedure Call/Return Instructions
16667 
16668 // Call Java Static Instruction
16669 
16670 instruct CallStaticJavaDirect(method meth)
16671 %{
16672   match(CallStaticJava);
16673 
16674   effect(USE meth);
16675 
16676   ins_cost(CALL_COST);
16677 
16678   format %{ "call,static $meth \t// ==> " %}
16679 
16680   ins_encode(aarch64_enc_java_static_call(meth),
16681              aarch64_enc_call_epilog);
16682 
16683   ins_pipe(pipe_class_call);
16684 %}
16685 
16686 // TO HERE
16687 
16688 // Call Java Dynamic Instruction
16689 instruct CallDynamicJavaDirect(method meth)
16690 %{
16691   match(CallDynamicJava);
16692 
16693   effect(USE meth);
16694 
16695   ins_cost(CALL_COST);
16696 
16697   format %{ "CALL,dynamic $meth \t// ==> " %}
16698 
16699   ins_encode(aarch64_enc_java_dynamic_call(meth),
16700              aarch64_enc_call_epilog);
16701 
16702   ins_pipe(pipe_class_call);
16703 %}
16704 
16705 // Call Runtime Instruction
16706 
16707 instruct CallRuntimeDirect(method meth)
16708 %{
16709   match(CallRuntime);
16710 
16711   effect(USE meth);
16712 
16713   ins_cost(CALL_COST);
16714 
16715   format %{ "CALL, runtime $meth" %}
16716 
16717   ins_encode( aarch64_enc_java_to_runtime(meth) );
16718 
16719   ins_pipe(pipe_class_call);
16720 %}
16721 
16722 // Call Runtime Instruction
16723 
16724 instruct CallLeafDirect(method meth)
16725 %{
16726   match(CallLeaf);
16727 
16728   effect(USE meth);
16729 
16730   ins_cost(CALL_COST);
16731 
16732   format %{ "CALL, runtime leaf $meth" %}
16733 
16734   ins_encode( aarch64_enc_java_to_runtime(meth) );
16735 
16736   ins_pipe(pipe_class_call);
16737 %}
16738 
16739 // Call Runtime Instruction
16740 
16741 instruct CallLeafNoFPDirect(method meth)
16742 %{
16743   match(CallLeafNoFP);
16744 
16745   effect(USE meth);
16746 
16747   ins_cost(CALL_COST);
16748 
16749   format %{ "CALL, runtime leaf nofp $meth" %}
16750 
16751   ins_encode( aarch64_enc_java_to_runtime(meth) );
16752 
16753   ins_pipe(pipe_class_call);
16754 %}
16755 
16756 instruct CallNativeDirect(method meth)
16757 %{
16758   match(CallNative);
16759 
16760   effect(USE meth);
16761 
16762   ins_cost(CALL_COST);
16763 
16764   format %{ "CALL, native $meth" %}
16765 
16766   ins_encode( aarch64_enc_java_to_runtime(meth) );
16767 
16768   ins_pipe(pipe_class_call);
16769 %}
16770 
16771 // Tail Call; Jump from runtime stub to Java code.
16772 // Also known as an 'interprocedural jump'.
16773 // Target of jump will eventually return to caller.
16774 // TailJump below removes the return address.
16775 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16776 %{
16777   match(TailCall jump_target method_ptr);
16778 
16779   ins_cost(CALL_COST);
16780 
16781   format %{ "br $jump_target\t# $method_ptr holds method" %}
16782 
16783   ins_encode(aarch64_enc_tail_call(jump_target));
16784 
16785   ins_pipe(pipe_class_call);
16786 %}
16787 
16788 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16789 %{
16790   match(TailJump jump_target ex_oop);
16791 
16792   ins_cost(CALL_COST);
16793 
16794   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16795 
16796   ins_encode(aarch64_enc_tail_jmp(jump_target));
16797 
16798   ins_pipe(pipe_class_call);
16799 %}
16800 
16801 // Create exception oop: created by stack-crawling runtime code.
16802 // Created exception is now available to this handler, and is setup
16803 // just prior to jumping to this handler. No code emitted.
16804 // TODO check
16805 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16806 instruct CreateException(iRegP_R0 ex_oop)
16807 %{
16808   match(Set ex_oop (CreateEx));
16809 
16810   format %{ " -- \t// exception oop; no code emitted" %}
16811 
16812   size(0);
16813 
16814   ins_encode( /*empty*/ );
16815 
16816   ins_pipe(pipe_class_empty);
16817 %}
16818 
16819 // Rethrow exception: The exception oop will come in the first
16820 // argument position. Then JUMP (not call) to the rethrow stub code.
16821 instruct RethrowException() %{
16822   match(Rethrow);
16823   ins_cost(CALL_COST);
16824 
16825   format %{ "b rethrow_stub" %}
16826 
16827   ins_encode( aarch64_enc_rethrow() );
16828 
16829   ins_pipe(pipe_class_call);
16830 %}
16831 
16832 
16833 // Return Instruction
16834 // epilog node loads ret address into lr as part of frame pop
16835 instruct Ret()
16836 %{
16837   match(Return);
16838 
16839   format %{ "ret\t// return register" %}
16840 
16841   ins_encode( aarch64_enc_ret() );
16842 
16843   ins_pipe(pipe_branch);
16844 %}
16845 
16846 // Die now.
16847 instruct ShouldNotReachHere() %{
16848   match(Halt);
16849 
16850   ins_cost(CALL_COST);
16851   format %{ "ShouldNotReachHere" %}
16852 
16853   ins_encode %{
16854     if (is_reachable()) {
16855       __ stop(_halt_reason);
16856     }
16857   %}
16858 
16859   ins_pipe(pipe_class_default);
16860 %}
16861 
16862 // ============================================================================
16863 // Partial Subtype Check
16864 //
16865 // superklass array for an instance of the superklass.  Set a hidden
16866 // internal cache on a hit (cache is checked with exposed code in
16867 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16868 // encoding ALSO sets flags.
16869 
16870 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16871 %{
16872   match(Set result (PartialSubtypeCheck sub super));
16873   effect(KILL cr, KILL temp);
16874 
16875   ins_cost(1100);  // slightly larger than the next version
16876   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16877 
16878   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16879 
16880   opcode(0x1); // Force zero of result reg on hit
16881 
16882   ins_pipe(pipe_class_memory);
16883 %}
16884 
16885 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16886 %{
16887   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16888   effect(KILL temp, KILL result);
16889 
16890   ins_cost(1100);  // slightly larger than the next version
16891   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16892 
16893   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16894 
16895   opcode(0x0); // Don't zero result reg on hit
16896 
16897   ins_pipe(pipe_class_memory);
16898 %}
16899 
16900 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16901                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16902 %{
16903   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16904   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16905   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16906 
16907   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16908   ins_encode %{
16909     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16910     __ string_compare($str1$$Register, $str2$$Register,
16911                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16912                       $tmp1$$Register, $tmp2$$Register,
16913                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16914   %}
16915   ins_pipe(pipe_class_memory);
16916 %}
16917 
16918 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16919                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16920 %{
16921   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16922   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16923   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16924 
16925   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16926   ins_encode %{
16927     __ string_compare($str1$$Register, $str2$$Register,
16928                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16929                       $tmp1$$Register, $tmp2$$Register,
16930                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16931   %}
16932   ins_pipe(pipe_class_memory);
16933 %}
16934 
16935 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16936                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16937                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16938 %{
16939   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16940   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16941   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16942          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16943 
16944   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16945   ins_encode %{
16946     __ string_compare($str1$$Register, $str2$$Register,
16947                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16948                       $tmp1$$Register, $tmp2$$Register,
16949                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16950                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16951   %}
16952   ins_pipe(pipe_class_memory);
16953 %}
16954 
16955 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16956                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16957                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16958 %{
16959   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16960   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16961   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16962          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16963 
16964   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16965   ins_encode %{
16966     __ string_compare($str1$$Register, $str2$$Register,
16967                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16968                       $tmp1$$Register, $tmp2$$Register,
16969                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16970                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16971   %}
16972   ins_pipe(pipe_class_memory);
16973 %}
16974 
16975 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16976        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16977        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16978 %{
16979   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16980   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16981   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16982          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16983   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16984 
16985   ins_encode %{
16986     __ string_indexof($str1$$Register, $str2$$Register,
16987                       $cnt1$$Register, $cnt2$$Register,
16988                       $tmp1$$Register, $tmp2$$Register,
16989                       $tmp3$$Register, $tmp4$$Register,
16990                       $tmp5$$Register, $tmp6$$Register,
16991                       -1, $result$$Register, StrIntrinsicNode::UU);
16992   %}
16993   ins_pipe(pipe_class_memory);
16994 %}
16995 
16996 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16997        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16998        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16999 %{
17000   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17001   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17002   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17003          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17004   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
17005 
17006   ins_encode %{
17007     __ string_indexof($str1$$Register, $str2$$Register,
17008                       $cnt1$$Register, $cnt2$$Register,
17009                       $tmp1$$Register, $tmp2$$Register,
17010                       $tmp3$$Register, $tmp4$$Register,
17011                       $tmp5$$Register, $tmp6$$Register,
17012                       -1, $result$$Register, StrIntrinsicNode::LL);
17013   %}
17014   ins_pipe(pipe_class_memory);
17015 %}
17016 
17017 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17018        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17019        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17020 %{
17021   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17022   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17023   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17024          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17025   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17026 
17027   ins_encode %{
17028     __ string_indexof($str1$$Register, $str2$$Register,
17029                       $cnt1$$Register, $cnt2$$Register,
17030                       $tmp1$$Register, $tmp2$$Register,
17031                       $tmp3$$Register, $tmp4$$Register,
17032                       $tmp5$$Register, $tmp6$$Register,
17033                       -1, $result$$Register, StrIntrinsicNode::UL);
17034   %}
17035   ins_pipe(pipe_class_memory);
17036 %}
17037 
17038 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17039                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17040                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17041 %{
17042   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17043   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17044   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17045          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17046   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17047 
17048   ins_encode %{
17049     int icnt2 = (int)$int_cnt2$$constant;
17050     __ string_indexof($str1$$Register, $str2$$Register,
17051                       $cnt1$$Register, zr,
17052                       $tmp1$$Register, $tmp2$$Register,
17053                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17054                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17055   %}
17056   ins_pipe(pipe_class_memory);
17057 %}
17058 
17059 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17060                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17061                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17062 %{
17063   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17064   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17065   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17066          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17067   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17068 
17069   ins_encode %{
17070     int icnt2 = (int)$int_cnt2$$constant;
17071     __ string_indexof($str1$$Register, $str2$$Register,
17072                       $cnt1$$Register, zr,
17073                       $tmp1$$Register, $tmp2$$Register,
17074                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17075                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17076   %}
17077   ins_pipe(pipe_class_memory);
17078 %}
17079 
17080 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17081                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17082                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17083 %{
17084   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17085   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17086   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17087          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17088   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17089 
17090   ins_encode %{
17091     int icnt2 = (int)$int_cnt2$$constant;
17092     __ string_indexof($str1$$Register, $str2$$Register,
17093                       $cnt1$$Register, zr,
17094                       $tmp1$$Register, $tmp2$$Register,
17095                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17096                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17097   %}
17098   ins_pipe(pipe_class_memory);
17099 %}
17100 
17101 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17102                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17103                              iRegINoSp tmp3, rFlagsReg cr)
17104 %{
17105   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17106   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17107   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17108          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17109 
17110   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17111 
17112   ins_encode %{
17113     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17114                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17115                            $tmp3$$Register);
17116   %}
17117   ins_pipe(pipe_class_memory);
17118 %}
17119 
17120 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17121                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17122                               iRegINoSp tmp3, rFlagsReg cr)
17123 %{
17124   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17125   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17126   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17127          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17128 
17129   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17130 
17131   ins_encode %{
17132     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17133                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17134                             $tmp3$$Register);
17135   %}
17136   ins_pipe(pipe_class_memory);
17137 %}
17138 
17139 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17140                         iRegI_R0 result, rFlagsReg cr)
17141 %{
17142   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17143   match(Set result (StrEquals (Binary str1 str2) cnt));
17144   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17145 
17146   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17147   ins_encode %{
17148     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17149     __ string_equals($str1$$Register, $str2$$Register,
17150                      $result$$Register, $cnt$$Register, 1);
17151   %}
17152   ins_pipe(pipe_class_memory);
17153 %}
17154 
17155 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17156                         iRegI_R0 result, rFlagsReg cr)
17157 %{
17158   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17159   match(Set result (StrEquals (Binary str1 str2) cnt));
17160   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17161 
17162   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17163   ins_encode %{
17164     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17165     __ string_equals($str1$$Register, $str2$$Register,
17166                      $result$$Register, $cnt$$Register, 2);
17167   %}
17168   ins_pipe(pipe_class_memory);
17169 %}
17170 
17171 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17172                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17173                        iRegP_R10 tmp, rFlagsReg cr)
17174 %{
17175   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17176   match(Set result (AryEq ary1 ary2));
17177   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17178 
17179   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17180   ins_encode %{
17181     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17182                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17183                                    $result$$Register, $tmp$$Register, 1);
17184     if (tpc == NULL) {
17185       ciEnv::current()->record_failure("CodeCache is full");
17186       return;
17187     }
17188   %}
17189   ins_pipe(pipe_class_memory);
17190 %}
17191 
17192 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17193                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17194                        iRegP_R10 tmp, rFlagsReg cr)
17195 %{
17196   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17197   match(Set result (AryEq ary1 ary2));
17198   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17199 
17200   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17201   ins_encode %{
17202     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17203                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17204                                    $result$$Register, $tmp$$Register, 2);
17205     if (tpc == NULL) {
17206       ciEnv::current()->record_failure("CodeCache is full");
17207       return;
17208     }
17209   %}
17210   ins_pipe(pipe_class_memory);
17211 %}
17212 
17213 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17214 %{
17215   match(Set result (CountPositives ary1 len));
17216   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17217   format %{ "count positives byte[] $ary1,$len -> $result" %}
17218   ins_encode %{
17219     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17220     if (tpc == NULL) {
17221       ciEnv::current()->record_failure("CodeCache is full");
17222       return;
17223     }
17224   %}
17225   ins_pipe( pipe_slow );
17226 %}
17227 
17228 // fast char[] to byte[] compression
17229 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17230                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17231                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17232                          iRegI_R0 result, rFlagsReg cr)
17233 %{
17234   match(Set result (StrCompressedCopy src (Binary dst len)));
17235   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17236          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17237 
17238   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
17239   ins_encode %{
17240     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17241                            $result$$Register,
17242                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17243                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17244   %}
17245   ins_pipe(pipe_slow);
17246 %}
17247 
17248 // fast byte[] to char[] inflation
17249 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17250                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17251 %{
17252   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17253   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17254 
17255   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17256   ins_encode %{
17257     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17258                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17259                                         $tmp3$$FloatRegister, $tmp4$$Register);
17260     if (tpc == NULL) {
17261       ciEnv::current()->record_failure("CodeCache is full");
17262       return;
17263     }
17264   %}
17265   ins_pipe(pipe_class_memory);
17266 %}
17267 
17268 // encode char[] to byte[] in ISO_8859_1
17269 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17270                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17271                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17272                           iRegI_R0 result, rFlagsReg cr)
17273 %{
17274   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17275   match(Set result (EncodeISOArray src (Binary dst len)));
17276   effect(USE_KILL src, USE_KILL dst, USE len,
17277          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17278 
17279   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17280   ins_encode %{
17281     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17282                         $result$$Register, false,
17283                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17284                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17285   %}
17286   ins_pipe(pipe_class_memory);
17287 %}
17288 
17289 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17290                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17291                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17292                             iRegI_R0 result, rFlagsReg cr)
17293 %{
17294   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17295   match(Set result (EncodeISOArray src (Binary dst len)));
17296   effect(USE_KILL src, USE_KILL dst, USE len,
17297          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17298 
17299   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17300   ins_encode %{
17301     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17302                         $result$$Register, true,
17303                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17304                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17305   %}
17306   ins_pipe(pipe_class_memory);
17307 %}
17308 
17309 // ============================================================================
17310 // This name is KNOWN by the ADLC and cannot be changed.
17311 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17312 // for this guy.
17313 instruct tlsLoadP(thread_RegP dst)
17314 %{
17315   match(Set dst (ThreadLocal));
17316 
17317   ins_cost(0);
17318 
17319   format %{ " -- \t// $dst=Thread::current(), empty" %}
17320 
17321   size(0);
17322 
17323   ins_encode( /*empty*/ );
17324 
17325   ins_pipe(pipe_class_empty);
17326 %}
17327 
17328 //----------PEEPHOLE RULES-----------------------------------------------------
17329 // These must follow all instruction definitions as they use the names
17330 // defined in the instructions definitions.
17331 //
17332 // peepmatch ( root_instr_name [preceding_instruction]* );
17333 //
17334 // peepconstraint %{
17335 // (instruction_number.operand_name relational_op instruction_number.operand_name
17336 //  [, ...] );
17337 // // instruction numbers are zero-based using left to right order in peepmatch
17338 //
17339 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17340 // // provide an instruction_number.operand_name for each operand that appears
17341 // // in the replacement instruction's match rule
17342 //
17343 // ---------VM FLAGS---------------------------------------------------------
17344 //
17345 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17346 //
17347 // Each peephole rule is given an identifying number starting with zero and
17348 // increasing by one in the order seen by the parser.  An individual peephole
17349 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17350 // on the command-line.
17351 //
17352 // ---------CURRENT LIMITATIONS----------------------------------------------
17353 //
17354 // Only match adjacent instructions in same basic block
17355 // Only equality constraints
17356 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17357 // Only one replacement instruction
17358 //
17359 // ---------EXAMPLE----------------------------------------------------------
17360 //
17361 // // pertinent parts of existing instructions in architecture description
17362 // instruct movI(iRegINoSp dst, iRegI src)
17363 // %{
17364 //   match(Set dst (CopyI src));
17365 // %}
17366 //
17367 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17368 // %{
17369 //   match(Set dst (AddI dst src));
17370 //   effect(KILL cr);
17371 // %}
17372 //
17373 // // Change (inc mov) to lea
17374 // peephole %{
17375 //   // increment preceded by register-register move
17376 //   peepmatch ( incI_iReg movI );
17377 //   // require that the destination register of the increment
17378 //   // match the destination register of the move
17379 //   peepconstraint ( 0.dst == 1.dst );
17380 //   // construct a replacement instruction that sets
17381 //   // the destination to ( move's source register + one )
17382 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17383 // %}
17384 //
17385 
17386 // Implementation no longer uses movX instructions since
17387 // machine-independent system no longer uses CopyX nodes.
17388 //
17389 // peephole
17390 // %{
17391 //   peepmatch (incI_iReg movI);
17392 //   peepconstraint (0.dst == 1.dst);
17393 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17394 // %}
17395 
17396 // peephole
17397 // %{
17398 //   peepmatch (decI_iReg movI);
17399 //   peepconstraint (0.dst == 1.dst);
17400 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17401 // %}
17402 
17403 // peephole
17404 // %{
17405 //   peepmatch (addI_iReg_imm movI);
17406 //   peepconstraint (0.dst == 1.dst);
17407 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17408 // %}
17409 
17410 // peephole
17411 // %{
17412 //   peepmatch (incL_iReg movL);
17413 //   peepconstraint (0.dst == 1.dst);
17414 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17415 // %}
17416 
17417 // peephole
17418 // %{
17419 //   peepmatch (decL_iReg movL);
17420 //   peepconstraint (0.dst == 1.dst);
17421 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17422 // %}
17423 
17424 // peephole
17425 // %{
17426 //   peepmatch (addL_iReg_imm movL);
17427 //   peepconstraint (0.dst == 1.dst);
17428 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17429 // %}
17430 
17431 // peephole
17432 // %{
17433 //   peepmatch (addP_iReg_imm movP);
17434 //   peepconstraint (0.dst == 1.dst);
17435 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17436 // %}
17437 
17438 // // Change load of spilled value to only a spill
17439 // instruct storeI(memory mem, iRegI src)
17440 // %{
17441 //   match(Set mem (StoreI mem src));
17442 // %}
17443 //
17444 // instruct loadI(iRegINoSp dst, memory mem)
17445 // %{
17446 //   match(Set dst (LoadI mem));
17447 // %}
17448 //
17449 
17450 //----------SMARTSPILL RULES---------------------------------------------------
17451 // These must follow all instruction definitions as they use the names
17452 // defined in the instructions definitions.
17453 
17454 // Local Variables:
17455 // mode: c++
17456 // End: