1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 reg_class p0_reg(P0);
 1203 reg_class p1_reg(P1);
 1204 
 1205 // Singleton class for condition codes
 1206 reg_class int_flags(RFLAGS);
 1207 
 1208 %}
 1209 
 1210 //----------DEFINITION BLOCK---------------------------------------------------
 1211 // Define name --> value mappings to inform the ADLC of an integer valued name
 1212 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1213 // Format:
 1214 //        int_def  <name>         ( <int_value>, <expression>);
 1215 // Generated Code in ad_<arch>.hpp
 1216 //        #define  <name>   (<expression>)
 1217 //        // value == <int_value>
 1218 // Generated code in ad_<arch>.cpp adlc_verification()
 1219 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1220 //
 1221 
 1222 // we follow the ppc-aix port in using a simple cost model which ranks
 1223 // register operations as cheap, memory ops as more expensive and
 1224 // branches as most expensive. the first two have a low as well as a
 1225 // normal cost. huge cost appears to be a way of saying don't do
 1226 // something
 1227 
 1228 definitions %{
 1229   // The default cost (of a register move instruction).
 1230   int_def INSN_COST            (    100,     100);
 1231   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1232   int_def CALL_COST            (    200,     2 * INSN_COST);
 1233   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1234 %}
 1235 
 1236 
 1237 //----------SOURCE BLOCK-------------------------------------------------------
 1238 // This is a block of C++ code which provides values, functions, and
 1239 // definitions necessary in the rest of the architecture description
 1240 
 1241 source_hpp %{
 1242 
 1243 #include "asm/macroAssembler.hpp"
 1244 #include "gc/shared/barrierSetAssembler.hpp"
 1245 #include "gc/shared/cardTable.hpp"
 1246 #include "gc/shared/cardTableBarrierSet.hpp"
 1247 #include "gc/shared/collectedHeap.hpp"
 1248 #include "opto/addnode.hpp"
 1249 #include "opto/convertnode.hpp"
 1250 #include "runtime/objectMonitor.hpp"
 1251 
 1252 extern RegMask _ANY_REG32_mask;
 1253 extern RegMask _ANY_REG_mask;
 1254 extern RegMask _PTR_REG_mask;
 1255 extern RegMask _NO_SPECIAL_REG32_mask;
 1256 extern RegMask _NO_SPECIAL_REG_mask;
 1257 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1258 
 1259 class CallStubImpl {
 1260 
 1261   //--------------------------------------------------------------
 1262   //---<  Used for optimization in Compile::shorten_branches  >---
 1263   //--------------------------------------------------------------
 1264 
 1265  public:
 1266   // Size of call trampoline stub.
 1267   static uint size_call_trampoline() {
 1268     return 0; // no call trampolines on this platform
 1269   }
 1270 
 1271   // number of relocations needed by a call trampoline stub
 1272   static uint reloc_call_trampoline() {
 1273     return 0; // no call trampolines on this platform
 1274   }
 1275 };
 1276 
 1277 class HandlerImpl {
 1278 
 1279  public:
 1280 
 1281   static int emit_exception_handler(CodeBuffer &cbuf);
 1282   static int emit_deopt_handler(CodeBuffer& cbuf);
 1283 
 1284   static uint size_exception_handler() {
 1285     return MacroAssembler::far_branch_size();
 1286   }
 1287 
 1288   static uint size_deopt_handler() {
 1289     // count one adr and one far branch instruction
 1290     return 4 * NativeInstruction::instruction_size;
 1291   }
 1292 };
 1293 
 1294 class Node::PD {
 1295 public:
 1296   enum NodeFlags {
 1297     _last_flag = Node::_last_flag
 1298   };
 1299 };
 1300 
 1301   bool is_CAS(int opcode, bool maybe_volatile);
 1302 
 1303   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1304 
 1305   bool unnecessary_acquire(const Node *barrier);
 1306   bool needs_acquiring_load(const Node *load);
 1307 
 1308   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1309 
 1310   bool unnecessary_release(const Node *barrier);
 1311   bool unnecessary_volatile(const Node *barrier);
 1312   bool needs_releasing_store(const Node *store);
 1313 
 1314   // predicate controlling translation of CompareAndSwapX
 1315   bool needs_acquiring_load_exclusive(const Node *load);
 1316 
 1317   // predicate controlling addressing modes
 1318   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1319 %}
 1320 
 1321 source %{
 1322 
 1323   // Derived RegMask with conditionally allocatable registers
 1324 
 1325   void PhaseOutput::pd_perform_mach_node_analysis() {
 1326   }
 1327 
 1328   int MachNode::pd_alignment_required() const {
 1329     return 1;
 1330   }
 1331 
 1332   int MachNode::compute_padding(int current_offset) const {
 1333     return 0;
 1334   }
 1335 
 1336   RegMask _ANY_REG32_mask;
 1337   RegMask _ANY_REG_mask;
 1338   RegMask _PTR_REG_mask;
 1339   RegMask _NO_SPECIAL_REG32_mask;
 1340   RegMask _NO_SPECIAL_REG_mask;
 1341   RegMask _NO_SPECIAL_PTR_REG_mask;
 1342 
 1343   void reg_mask_init() {
 1344     // We derive below RegMask(s) from the ones which are auto-generated from
 1345     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1346     // registers conditionally reserved.
 1347 
 1348     _ANY_REG32_mask = _ALL_REG32_mask;
 1349     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1350 
 1351     _ANY_REG_mask = _ALL_REG_mask;
 1352 
 1353     _PTR_REG_mask = _ALL_REG_mask;
 1354 
 1355     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1356     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1357 
 1358     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1362     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1363 
 1364     // r27 is not allocatable when compressed oops is on and heapbase is not
 1365     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1366     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1367       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1368       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1369       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1370     }
 1371 
 1372     // r29 is not allocatable when PreserveFramePointer is on
 1373     if (PreserveFramePointer) {
 1374       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1375       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1376       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1377     }
 1378   }
 1379 
 1380   // Optimizaton of volatile gets and puts
 1381   // -------------------------------------
 1382   //
 1383   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1384   // use to implement volatile reads and writes. For a volatile read
 1385   // we simply need
 1386   //
 1387   //   ldar<x>
 1388   //
 1389   // and for a volatile write we need
 1390   //
 1391   //   stlr<x>
 1392   //
 1393   // Alternatively, we can implement them by pairing a normal
 1394   // load/store with a memory barrier. For a volatile read we need
 1395   //
 1396   //   ldr<x>
 1397   //   dmb ishld
 1398   //
 1399   // for a volatile write
 1400   //
 1401   //   dmb ish
 1402   //   str<x>
 1403   //   dmb ish
 1404   //
 1405   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1406   // sequences. These are normally translated to an instruction
 1407   // sequence like the following
 1408   //
 1409   //   dmb      ish
 1410   // retry:
 1411   //   ldxr<x>   rval raddr
 1412   //   cmp       rval rold
 1413   //   b.ne done
 1414   //   stlxr<x>  rval, rnew, rold
 1415   //   cbnz      rval retry
 1416   // done:
 1417   //   cset      r0, eq
 1418   //   dmb ishld
 1419   //
 1420   // Note that the exclusive store is already using an stlxr
 1421   // instruction. That is required to ensure visibility to other
 1422   // threads of the exclusive write (assuming it succeeds) before that
 1423   // of any subsequent writes.
 1424   //
 1425   // The following instruction sequence is an improvement on the above
 1426   //
 1427   // retry:
 1428   //   ldaxr<x>  rval raddr
 1429   //   cmp       rval rold
 1430   //   b.ne done
 1431   //   stlxr<x>  rval, rnew, rold
 1432   //   cbnz      rval retry
 1433   // done:
 1434   //   cset      r0, eq
 1435   //
 1436   // We don't need the leading dmb ish since the stlxr guarantees
 1437   // visibility of prior writes in the case that the swap is
 1438   // successful. Crucially we don't have to worry about the case where
 1439   // the swap is not successful since no valid program should be
 1440   // relying on visibility of prior changes by the attempting thread
 1441   // in the case where the CAS fails.
 1442   //
 1443   // Similarly, we don't need the trailing dmb ishld if we substitute
 1444   // an ldaxr instruction since that will provide all the guarantees we
 1445   // require regarding observation of changes made by other threads
 1446   // before any change to the CAS address observed by the load.
 1447   //
 1448   // In order to generate the desired instruction sequence we need to
 1449   // be able to identify specific 'signature' ideal graph node
 1450   // sequences which i) occur as a translation of a volatile reads or
 1451   // writes or CAS operations and ii) do not occur through any other
 1452   // translation or graph transformation. We can then provide
 1453   // alternative aldc matching rules which translate these node
 1454   // sequences to the desired machine code sequences. Selection of the
 1455   // alternative rules can be implemented by predicates which identify
 1456   // the relevant node sequences.
 1457   //
 1458   // The ideal graph generator translates a volatile read to the node
 1459   // sequence
 1460   //
 1461   //   LoadX[mo_acquire]
 1462   //   MemBarAcquire
 1463   //
 1464   // As a special case when using the compressed oops optimization we
 1465   // may also see this variant
 1466   //
 1467   //   LoadN[mo_acquire]
 1468   //   DecodeN
 1469   //   MemBarAcquire
 1470   //
 1471   // A volatile write is translated to the node sequence
 1472   //
 1473   //   MemBarRelease
 1474   //   StoreX[mo_release] {CardMark}-optional
 1475   //   MemBarVolatile
 1476   //
 1477   // n.b. the above node patterns are generated with a strict
 1478   // 'signature' configuration of input and output dependencies (see
 1479   // the predicates below for exact details). The card mark may be as
 1480   // simple as a few extra nodes or, in a few GC configurations, may
 1481   // include more complex control flow between the leading and
 1482   // trailing memory barriers. However, whatever the card mark
 1483   // configuration these signatures are unique to translated volatile
 1484   // reads/stores -- they will not appear as a result of any other
 1485   // bytecode translation or inlining nor as a consequence of
 1486   // optimizing transforms.
 1487   //
 1488   // We also want to catch inlined unsafe volatile gets and puts and
 1489   // be able to implement them using either ldar<x>/stlr<x> or some
 1490   // combination of ldr<x>/stlr<x> and dmb instructions.
 1491   //
 1492   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1493   // normal volatile put node sequence containing an extra cpuorder
 1494   // membar
 1495   //
 1496   //   MemBarRelease
 1497   //   MemBarCPUOrder
 1498   //   StoreX[mo_release] {CardMark}-optional
 1499   //   MemBarCPUOrder
 1500   //   MemBarVolatile
 1501   //
 1502   // n.b. as an aside, a cpuorder membar is not itself subject to
 1503   // matching and translation by adlc rules.  However, the rule
 1504   // predicates need to detect its presence in order to correctly
 1505   // select the desired adlc rules.
 1506   //
 1507   // Inlined unsafe volatile gets manifest as a slightly different
 1508   // node sequence to a normal volatile get because of the
 1509   // introduction of some CPUOrder memory barriers to bracket the
 1510   // Load. However, but the same basic skeleton of a LoadX feeding a
 1511   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1512   // present
 1513   //
 1514   //   MemBarCPUOrder
 1515   //        ||       \\
 1516   //   MemBarCPUOrder LoadX[mo_acquire]
 1517   //        ||            |
 1518   //        ||       {DecodeN} optional
 1519   //        ||       /
 1520   //     MemBarAcquire
 1521   //
 1522   // In this case the acquire membar does not directly depend on the
 1523   // load. However, we can be sure that the load is generated from an
 1524   // inlined unsafe volatile get if we see it dependent on this unique
 1525   // sequence of membar nodes. Similarly, given an acquire membar we
 1526   // can know that it was added because of an inlined unsafe volatile
 1527   // get if it is fed and feeds a cpuorder membar and if its feed
 1528   // membar also feeds an acquiring load.
 1529   //
 1530   // Finally an inlined (Unsafe) CAS operation is translated to the
 1531   // following ideal graph
 1532   //
 1533   //   MemBarRelease
 1534   //   MemBarCPUOrder
 1535   //   CompareAndSwapX {CardMark}-optional
 1536   //   MemBarCPUOrder
 1537   //   MemBarAcquire
 1538   //
 1539   // So, where we can identify these volatile read and write
 1540   // signatures we can choose to plant either of the above two code
 1541   // sequences. For a volatile read we can simply plant a normal
 1542   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1543   // also choose to inhibit translation of the MemBarAcquire and
 1544   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1545   //
 1546   // When we recognise a volatile store signature we can choose to
 1547   // plant at a dmb ish as a translation for the MemBarRelease, a
 1548   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1549   // Alternatively, we can inhibit translation of the MemBarRelease
 1550   // and MemBarVolatile and instead plant a simple stlr<x>
 1551   // instruction.
 1552   //
 1553   // when we recognise a CAS signature we can choose to plant a dmb
 1554   // ish as a translation for the MemBarRelease, the conventional
 1555   // macro-instruction sequence for the CompareAndSwap node (which
 1556   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1557   // Alternatively, we can elide generation of the dmb instructions
 1558   // and plant the alternative CompareAndSwap macro-instruction
 1559   // sequence (which uses ldaxr<x>).
 1560   //
 1561   // Of course, the above only applies when we see these signature
 1562   // configurations. We still want to plant dmb instructions in any
 1563   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1564   // MemBarVolatile. For example, at the end of a constructor which
 1565   // writes final/volatile fields we will see a MemBarRelease
 1566   // instruction and this needs a 'dmb ish' lest we risk the
 1567   // constructed object being visible without making the
 1568   // final/volatile field writes visible.
 1569   //
 1570   // n.b. the translation rules below which rely on detection of the
 1571   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1572   // If we see anything other than the signature configurations we
 1573   // always just translate the loads and stores to ldr<x> and str<x>
 1574   // and translate acquire, release and volatile membars to the
 1575   // relevant dmb instructions.
 1576   //
 1577 
 1578   // is_CAS(int opcode, bool maybe_volatile)
 1579   //
 1580   // return true if opcode is one of the possible CompareAndSwapX
 1581   // values otherwise false.
 1582 
 1583   bool is_CAS(int opcode, bool maybe_volatile)
 1584   {
 1585     switch(opcode) {
 1586       // We handle these
 1587     case Op_CompareAndSwapI:
 1588     case Op_CompareAndSwapL:
 1589     case Op_CompareAndSwapP:
 1590     case Op_CompareAndSwapN:
 1591     case Op_ShenandoahCompareAndSwapP:
 1592     case Op_ShenandoahCompareAndSwapN:
 1593     case Op_CompareAndSwapB:
 1594     case Op_CompareAndSwapS:
 1595     case Op_GetAndSetI:
 1596     case Op_GetAndSetL:
 1597     case Op_GetAndSetP:
 1598     case Op_GetAndSetN:
 1599     case Op_GetAndAddI:
 1600     case Op_GetAndAddL:
 1601       return true;
 1602     case Op_CompareAndExchangeI:
 1603     case Op_CompareAndExchangeN:
 1604     case Op_CompareAndExchangeB:
 1605     case Op_CompareAndExchangeS:
 1606     case Op_CompareAndExchangeL:
 1607     case Op_CompareAndExchangeP:
 1608     case Op_WeakCompareAndSwapB:
 1609     case Op_WeakCompareAndSwapS:
 1610     case Op_WeakCompareAndSwapI:
 1611     case Op_WeakCompareAndSwapL:
 1612     case Op_WeakCompareAndSwapP:
 1613     case Op_WeakCompareAndSwapN:
 1614     case Op_ShenandoahWeakCompareAndSwapP:
 1615     case Op_ShenandoahWeakCompareAndSwapN:
 1616     case Op_ShenandoahCompareAndExchangeP:
 1617     case Op_ShenandoahCompareAndExchangeN:
 1618       return maybe_volatile;
 1619     default:
 1620       return false;
 1621     }
 1622   }
 1623 
 1624   // helper to determine the maximum number of Phi nodes we may need to
 1625   // traverse when searching from a card mark membar for the merge mem
 1626   // feeding a trailing membar or vice versa
 1627 
 1628 // predicates controlling emit of ldr<x>/ldar<x>
 1629 
 1630 bool unnecessary_acquire(const Node *barrier)
 1631 {
 1632   assert(barrier->is_MemBar(), "expecting a membar");
 1633 
 1634   MemBarNode* mb = barrier->as_MemBar();
 1635 
 1636   if (mb->trailing_load()) {
 1637     return true;
 1638   }
 1639 
 1640   if (mb->trailing_load_store()) {
 1641     Node* load_store = mb->in(MemBarNode::Precedent);
 1642     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1643     return is_CAS(load_store->Opcode(), true);
 1644   }
 1645 
 1646   return false;
 1647 }
 1648 
 1649 bool needs_acquiring_load(const Node *n)
 1650 {
 1651   assert(n->is_Load(), "expecting a load");
 1652   LoadNode *ld = n->as_Load();
 1653   return ld->is_acquire();
 1654 }
 1655 
 1656 bool unnecessary_release(const Node *n)
 1657 {
 1658   assert((n->is_MemBar() &&
 1659           n->Opcode() == Op_MemBarRelease),
 1660          "expecting a release membar");
 1661 
 1662   MemBarNode *barrier = n->as_MemBar();
 1663   if (!barrier->leading()) {
 1664     return false;
 1665   } else {
 1666     Node* trailing = barrier->trailing_membar();
 1667     MemBarNode* trailing_mb = trailing->as_MemBar();
 1668     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1669     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1670 
 1671     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1672     if (mem->is_Store()) {
 1673       assert(mem->as_Store()->is_release(), "");
 1674       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1675       return true;
 1676     } else {
 1677       assert(mem->is_LoadStore(), "");
 1678       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1679       return is_CAS(mem->Opcode(), true);
 1680     }
 1681   }
 1682   return false;
 1683 }
 1684 
 1685 bool unnecessary_volatile(const Node *n)
 1686 {
 1687   // assert n->is_MemBar();
 1688   MemBarNode *mbvol = n->as_MemBar();
 1689 
 1690   bool release = mbvol->trailing_store();
 1691   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1692 #ifdef ASSERT
 1693   if (release) {
 1694     Node* leading = mbvol->leading_membar();
 1695     assert(leading->Opcode() == Op_MemBarRelease, "");
 1696     assert(leading->as_MemBar()->leading_store(), "");
 1697     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1698   }
 1699 #endif
 1700 
 1701   return release;
 1702 }
 1703 
 1704 // predicates controlling emit of str<x>/stlr<x>
 1705 
 1706 bool needs_releasing_store(const Node *n)
 1707 {
 1708   // assert n->is_Store();
 1709   StoreNode *st = n->as_Store();
 1710   return st->trailing_membar() != NULL;
 1711 }
 1712 
 1713 // predicate controlling translation of CAS
 1714 //
 1715 // returns true if CAS needs to use an acquiring load otherwise false
 1716 
 1717 bool needs_acquiring_load_exclusive(const Node *n)
 1718 {
 1719   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1720   LoadStoreNode* ldst = n->as_LoadStore();
 1721   if (is_CAS(n->Opcode(), false)) {
 1722     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1723   } else {
 1724     return ldst->trailing_membar() != NULL;
 1725   }
 1726 
 1727   // so we can just return true here
 1728   return true;
 1729 }
 1730 
 1731 #define __ _masm.
 1732 
 1733 // advance declarations for helper functions to convert register
 1734 // indices to register objects
 1735 
 1736 // the ad file has to provide implementations of certain methods
 1737 // expected by the generic code
 1738 //
 1739 // REQUIRED FUNCTIONALITY
 1740 
 1741 //=============================================================================
 1742 
 1743 // !!!!! Special hack to get all types of calls to specify the byte offset
 1744 //       from the start of the call to the point where the return address
 1745 //       will point.
 1746 
 1747 int MachCallStaticJavaNode::ret_addr_offset()
 1748 {
 1749   // call should be a simple bl
 1750   int off = 4;
 1751   return off;
 1752 }
 1753 
 1754 int MachCallDynamicJavaNode::ret_addr_offset()
 1755 {
 1756   return 16; // movz, movk, movk, bl
 1757 }
 1758 
 1759 int MachCallRuntimeNode::ret_addr_offset() {
 1760   // for generated stubs the call will be
 1761   //   bl(addr)
 1762   // or with far branches
 1763   //   bl(trampoline_stub)
 1764   // for real runtime callouts it will be six instructions
 1765   // see aarch64_enc_java_to_runtime
 1766   //   adr(rscratch2, retaddr)
 1767   //   lea(rscratch1, RuntimeAddress(addr)
 1768   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1769   //   blr(rscratch1)
 1770   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1771   if (cb) {
 1772     return 1 * NativeInstruction::instruction_size;
 1773   } else {
 1774     return 6 * NativeInstruction::instruction_size;
 1775   }
 1776 }
 1777 
 1778 int MachCallNativeNode::ret_addr_offset() {
 1779   // This is implemented using aarch64_enc_java_to_runtime as above.
 1780   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1781   if (cb) {
 1782     return 1 * NativeInstruction::instruction_size;
 1783   } else {
 1784     return 6 * NativeInstruction::instruction_size;
 1785   }
 1786 }
 1787 
 1788 //=============================================================================
 1789 
 1790 #ifndef PRODUCT
 1791 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1792   st->print("BREAKPOINT");
 1793 }
 1794 #endif
 1795 
 1796 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1797   C2_MacroAssembler _masm(&cbuf);
 1798   __ brk(0);
 1799 }
 1800 
 1801 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1802   return MachNode::size(ra_);
 1803 }
 1804 
 1805 //=============================================================================
 1806 
 1807 #ifndef PRODUCT
 1808   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1809     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1810   }
 1811 #endif
 1812 
 1813   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1814     C2_MacroAssembler _masm(&cbuf);
 1815     for (int i = 0; i < _count; i++) {
 1816       __ nop();
 1817     }
 1818   }
 1819 
 1820   uint MachNopNode::size(PhaseRegAlloc*) const {
 1821     return _count * NativeInstruction::instruction_size;
 1822   }
 1823 
 1824 //=============================================================================
 1825 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1826 
 1827 int ConstantTable::calculate_table_base_offset() const {
 1828   return 0;  // absolute addressing, no offset
 1829 }
 1830 
 1831 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1832 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1833   ShouldNotReachHere();
 1834 }
 1835 
 1836 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1837   // Empty encoding
 1838 }
 1839 
 1840 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1841   return 0;
 1842 }
 1843 
 1844 #ifndef PRODUCT
 1845 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1846   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1847 }
 1848 #endif
 1849 
 1850 #ifndef PRODUCT
 1851 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1852   Compile* C = ra_->C;
 1853 
 1854   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1855 
 1856   if (C->output()->need_stack_bang(framesize))
 1857     st->print("# stack bang size=%d\n\t", framesize);
 1858 
 1859   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1860     st->print("sub  sp, sp, #%d\n\t", framesize);
 1861     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1862     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1863   } else {
 1864     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1865     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1866     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1867     st->print("sub  sp, sp, rscratch1");
 1868   }
 1869   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1870     st->print("\n\t");
 1871     st->print("ldr  rscratch1, [guard]\n\t");
 1872     st->print("dmb ishld\n\t");
 1873     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1874     st->print("cmp  rscratch1, rscratch2\n\t");
 1875     st->print("b.eq skip");
 1876     st->print("\n\t");
 1877     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1878     st->print("b skip\n\t");
 1879     st->print("guard: int\n\t");
 1880     st->print("\n\t");
 1881     st->print("skip:\n\t");
 1882   }
 1883 }
 1884 #endif
 1885 
 1886 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1887   Compile* C = ra_->C;
 1888   C2_MacroAssembler _masm(&cbuf);
 1889 
 1890   // n.b. frame size includes space for return pc and rfp
 1891   const int framesize = C->output()->frame_size_in_bytes();
 1892 
 1893   // insert a nop at the start of the prolog so we can patch in a
 1894   // branch if we need to invalidate the method later
 1895   __ nop();
 1896 
 1897   if (C->clinit_barrier_on_entry()) {
 1898     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1899 
 1900     Label L_skip_barrier;
 1901 
 1902     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1903     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1904     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1905     __ bind(L_skip_barrier);
 1906   }
 1907 
 1908   if (C->max_vector_size() > 0) {
 1909     __ reinitialize_ptrue();
 1910   }
 1911 
 1912   int bangsize = C->output()->bang_size_in_bytes();
 1913   if (C->output()->need_stack_bang(bangsize))
 1914     __ generate_stack_overflow_check(bangsize);
 1915 
 1916   __ build_frame(framesize);
 1917 
 1918   if (C->stub_function() == NULL) {
 1919     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1920     bs->nmethod_entry_barrier(&_masm);
 1921   }
 1922 
 1923   if (VerifyStackAtCalls) {
 1924     Unimplemented();
 1925   }
 1926 
 1927   C->output()->set_frame_complete(cbuf.insts_size());
 1928 
 1929   if (C->has_mach_constant_base_node()) {
 1930     // NOTE: We set the table base offset here because users might be
 1931     // emitted before MachConstantBaseNode.
 1932     ConstantTable& constant_table = C->output()->constant_table();
 1933     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1934   }
 1935 }
 1936 
 1937 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1938 {
 1939   return MachNode::size(ra_); // too many variables; just compute it
 1940                               // the hard way
 1941 }
 1942 
 1943 int MachPrologNode::reloc() const
 1944 {
 1945   return 0;
 1946 }
 1947 
 1948 //=============================================================================
 1949 
 1950 #ifndef PRODUCT
 1951 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1952   Compile* C = ra_->C;
 1953   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1954 
 1955   st->print("# pop frame %d\n\t",framesize);
 1956 
 1957   if (framesize == 0) {
 1958     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1959   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1960     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, #%d\n\t", framesize);
 1962   } else {
 1963     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1964     st->print("add  sp, sp, rscratch1\n\t");
 1965     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1966   }
 1967 
 1968   if (do_polling() && C->is_method_compilation()) {
 1969     st->print("# test polling word\n\t");
 1970     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1971     st->print("cmp  sp, rscratch1\n\t");
 1972     st->print("bhi #slow_path");
 1973   }
 1974 }
 1975 #endif
 1976 
 1977 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1978   Compile* C = ra_->C;
 1979   C2_MacroAssembler _masm(&cbuf);
 1980   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1981 
 1982   __ remove_frame(framesize);
 1983 
 1984   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1985     __ reserved_stack_check();
 1986   }
 1987 
 1988   if (do_polling() && C->is_method_compilation()) {
 1989     Label dummy_label;
 1990     Label* code_stub = &dummy_label;
 1991     if (!C->output()->in_scratch_emit_size()) {
 1992       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1993     }
 1994     __ relocate(relocInfo::poll_return_type);
 1995     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1996   }
 1997 }
 1998 
 1999 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 2000   // Variable size. Determine dynamically.
 2001   return MachNode::size(ra_);
 2002 }
 2003 
 2004 int MachEpilogNode::reloc() const {
 2005   // Return number of relocatable values contained in this instruction.
 2006   return 1; // 1 for polling page.
 2007 }
 2008 
 2009 const Pipeline * MachEpilogNode::pipeline() const {
 2010   return MachNode::pipeline_class();
 2011 }
 2012 
 2013 //=============================================================================
 2014 
 2015 // Figure out which register class each belongs in: rc_int, rc_float or
 2016 // rc_stack.
 2017 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2018 
 2019 static enum RC rc_class(OptoReg::Name reg) {
 2020 
 2021   if (reg == OptoReg::Bad) {
 2022     return rc_bad;
 2023   }
 2024 
 2025   // we have 32 int registers * 2 halves
 2026   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2027 
 2028   if (reg < slots_of_int_registers) {
 2029     return rc_int;
 2030   }
 2031 
 2032   // we have 32 float register * 8 halves
 2033   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2034   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2035     return rc_float;
 2036   }
 2037 
 2038   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2039   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2040     return rc_predicate;
 2041   }
 2042 
 2043   // Between predicate regs & stack is the flags.
 2044   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2045 
 2046   return rc_stack;
 2047 }
 2048 
 2049 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2050   Compile* C = ra_->C;
 2051 
 2052   // Get registers to move.
 2053   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2054   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2055   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2056   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2057 
 2058   enum RC src_hi_rc = rc_class(src_hi);
 2059   enum RC src_lo_rc = rc_class(src_lo);
 2060   enum RC dst_hi_rc = rc_class(dst_hi);
 2061   enum RC dst_lo_rc = rc_class(dst_lo);
 2062 
 2063   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2064 
 2065   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 2066     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2067            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2068            "expected aligned-adjacent pairs");
 2069   }
 2070 
 2071   if (src_lo == dst_lo && src_hi == dst_hi) {
 2072     return 0;            // Self copy, no move.
 2073   }
 2074 
 2075   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2076               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2077   int src_offset = ra_->reg2offset(src_lo);
 2078   int dst_offset = ra_->reg2offset(dst_lo);
 2079 
 2080   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2081     uint ireg = ideal_reg();
 2082     if (ireg == Op_VecA && cbuf) {
 2083       C2_MacroAssembler _masm(cbuf);
 2084       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2085       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2086         // stack->stack
 2087         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2088                                                 sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2090         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2091                             sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2093         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2094                               sve_vector_reg_size_in_bytes);
 2095       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2096         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2097                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2098                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2099       } else {
 2100         ShouldNotReachHere();
 2101       }
 2102     } else if (cbuf) {
 2103       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2104       C2_MacroAssembler _masm(cbuf);
 2105       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2106       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2107         // stack->stack
 2108         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2109         if (ireg == Op_VecD) {
 2110           __ unspill(rscratch1, true, src_offset);
 2111           __ spill(rscratch1, true, dst_offset);
 2112         } else {
 2113           __ spill_copy128(src_offset, dst_offset);
 2114         }
 2115       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2116         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2117                ireg == Op_VecD ? __ T8B : __ T16B,
 2118                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2119       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2120         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2121                  ireg == Op_VecD ? __ D : __ Q,
 2122                  ra_->reg2offset(dst_lo));
 2123       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2124         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2125                    ireg == Op_VecD ? __ D : __ Q,
 2126                    ra_->reg2offset(src_lo));
 2127       } else {
 2128         ShouldNotReachHere();
 2129       }
 2130     }
 2131   } else if (cbuf) {
 2132     C2_MacroAssembler _masm(cbuf);
 2133     switch (src_lo_rc) {
 2134     case rc_int:
 2135       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2136         if (is64) {
 2137             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2138                    as_Register(Matcher::_regEncode[src_lo]));
 2139         } else {
 2140             C2_MacroAssembler _masm(cbuf);
 2141             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2142                     as_Register(Matcher::_regEncode[src_lo]));
 2143         }
 2144       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2145         if (is64) {
 2146             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2147                      as_Register(Matcher::_regEncode[src_lo]));
 2148         } else {
 2149             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2150                      as_Register(Matcher::_regEncode[src_lo]));
 2151         }
 2152       } else {                    // gpr --> stack spill
 2153         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2154         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2155       }
 2156       break;
 2157     case rc_float:
 2158       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2159         if (is64) {
 2160             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2161                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2162         } else {
 2163             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2164                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2165         }
 2166       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2167         if (is64) {
 2168             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         } else {
 2171             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2172                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2173         }
 2174       } else {                    // fpr --> stack spill
 2175         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2176         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2177                  is64 ? __ D : __ S, dst_offset);
 2178       }
 2179       break;
 2180     case rc_stack:
 2181       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2182         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2183       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2184         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2185                    is64 ? __ D : __ S, src_offset);
 2186       } else if (dst_lo_rc == rc_predicate) {
 2187         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2188                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2189       } else {                    // stack --> stack copy
 2190         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2191         if (ideal_reg() == Op_RegVectMask) {
 2192           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2193                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2194         } else {
 2195           __ unspill(rscratch1, is64, src_offset);
 2196           __ spill(rscratch1, is64, dst_offset);
 2197         }
 2198       }
 2199       break;
 2200     case rc_predicate:
 2201       if (dst_lo_rc == rc_predicate) {
 2202         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2203       } else if (dst_lo_rc == rc_stack) {
 2204         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2205                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2206       } else {
 2207         assert(false, "bad src and dst rc_class combination.");
 2208         ShouldNotReachHere();
 2209       }
 2210       break;
 2211     default:
 2212       assert(false, "bad rc_class for spill");
 2213       ShouldNotReachHere();
 2214     }
 2215   }
 2216 
 2217   if (st) {
 2218     st->print("spill ");
 2219     if (src_lo_rc == rc_stack) {
 2220       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2221     } else {
 2222       st->print("%s -> ", Matcher::regName[src_lo]);
 2223     }
 2224     if (dst_lo_rc == rc_stack) {
 2225       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2226     } else {
 2227       st->print("%s", Matcher::regName[dst_lo]);
 2228     }
 2229     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2230       int vsize = 0;
 2231       switch (ideal_reg()) {
 2232       case Op_VecD:
 2233         vsize = 64;
 2234         break;
 2235       case Op_VecX:
 2236         vsize = 128;
 2237         break;
 2238       case Op_VecA:
 2239         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2240         break;
 2241       default:
 2242         assert(false, "bad register type for spill");
 2243         ShouldNotReachHere();
 2244       }
 2245       st->print("\t# vector spill size = %d", vsize);
 2246     } else if (ideal_reg() == Op_RegVectMask) {
 2247       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2248       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2249       st->print("\t# predicate spill size = %d", vsize);
 2250     } else {
 2251       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2252     }
 2253   }
 2254 
 2255   return 0;
 2256 
 2257 }
 2258 
 2259 #ifndef PRODUCT
 2260 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2261   if (!ra_)
 2262     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2263   else
 2264     implementation(NULL, ra_, false, st);
 2265 }
 2266 #endif
 2267 
 2268 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2269   implementation(&cbuf, ra_, false, NULL);
 2270 }
 2271 
 2272 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2273   return MachNode::size(ra_);
 2274 }
 2275 
 2276 //=============================================================================
 2277 
 2278 #ifndef PRODUCT
 2279 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2280   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2281   int reg = ra_->get_reg_first(this);
 2282   st->print("add %s, rsp, #%d]\t# box lock",
 2283             Matcher::regName[reg], offset);
 2284 }
 2285 #endif
 2286 
 2287 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2288   C2_MacroAssembler _masm(&cbuf);
 2289 
 2290   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2291   int reg    = ra_->get_encode(this);
 2292 
 2293   // This add will handle any 24-bit signed offset. 24 bits allows an
 2294   // 8 megabyte stack frame.
 2295   __ add(as_Register(reg), sp, offset);
 2296 }
 2297 
 2298 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2299   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2300   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2301 
 2302   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2303     return NativeInstruction::instruction_size;
 2304   } else {
 2305     return 2 * NativeInstruction::instruction_size;
 2306   }
 2307 }
 2308 
 2309 //=============================================================================
 2310 
 2311 #ifndef PRODUCT
 2312 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2313 {
 2314   st->print_cr("# MachUEPNode");
 2315   if (UseCompressedClassPointers) {
 2316     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2317     if (CompressedKlassPointers::shift() != 0) {
 2318       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2319     }
 2320   } else {
 2321    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2322   }
 2323   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2324   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2325 }
 2326 #endif
 2327 
 2328 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2329 {
 2330   // This is the unverified entry point.
 2331   C2_MacroAssembler _masm(&cbuf);
 2332 
 2333   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2334   Label skip;
 2335   // TODO
 2336   // can we avoid this skip and still use a reloc?
 2337   __ br(Assembler::EQ, skip);
 2338   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2339   __ bind(skip);
 2340 }
 2341 
 2342 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2343 {
 2344   return MachNode::size(ra_);
 2345 }
 2346 
 2347 // REQUIRED EMIT CODE
 2348 
 2349 //=============================================================================
 2350 
 2351 // Emit exception handler code.
 2352 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2353 {
 2354   // mov rscratch1 #exception_blob_entry_point
 2355   // br rscratch1
 2356   // Note that the code buffer's insts_mark is always relative to insts.
 2357   // That's why we must use the macroassembler to generate a handler.
 2358   C2_MacroAssembler _masm(&cbuf);
 2359   address base = __ start_a_stub(size_exception_handler());
 2360   if (base == NULL) {
 2361     ciEnv::current()->record_failure("CodeCache is full");
 2362     return 0;  // CodeBuffer::expand failed
 2363   }
 2364   int offset = __ offset();
 2365   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2366   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2367   __ end_a_stub();
 2368   return offset;
 2369 }
 2370 
 2371 // Emit deopt handler code.
 2372 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2373 {
 2374   // Note that the code buffer's insts_mark is always relative to insts.
 2375   // That's why we must use the macroassembler to generate a handler.
 2376   C2_MacroAssembler _masm(&cbuf);
 2377   address base = __ start_a_stub(size_deopt_handler());
 2378   if (base == NULL) {
 2379     ciEnv::current()->record_failure("CodeCache is full");
 2380     return 0;  // CodeBuffer::expand failed
 2381   }
 2382   int offset = __ offset();
 2383 
 2384   __ adr(lr, __ pc());
 2385   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2386 
 2387   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
 2388   __ end_a_stub();
 2389   return offset;
 2390 }
 2391 
 2392 // REQUIRED MATCHER CODE
 2393 
 2394 //=============================================================================
 2395 
 2396 const bool Matcher::match_rule_supported(int opcode) {
 2397   if (!has_match_rule(opcode))
 2398     return false;
 2399 
 2400   bool ret_value = true;
 2401   switch (opcode) {
 2402     case Op_OnSpinWait:
 2403       return VM_Version::supports_on_spin_wait();
 2404     case Op_CacheWB:
 2405     case Op_CacheWBPreSync:
 2406     case Op_CacheWBPostSync:
 2407       if (!VM_Version::supports_data_cache_line_flush()) {
 2408         ret_value = false;
 2409       }
 2410       break;
 2411     case Op_LoadVectorMasked:
 2412     case Op_StoreVectorMasked:
 2413     case Op_LoadVectorGatherMasked:
 2414     case Op_StoreVectorScatterMasked:
 2415     case Op_MaskAll:
 2416     case Op_AndVMask:
 2417     case Op_OrVMask:
 2418     case Op_XorVMask:
 2419       if (UseSVE == 0) {
 2420         ret_value = false;
 2421       }
 2422       break;
 2423   }
 2424 
 2425   return ret_value; // Per default match rules are supported.
 2426 }
 2427 
 2428 // Identify extra cases that we might want to provide match rules for vector nodes and
 2429 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2430 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2431   if (!match_rule_supported(opcode)) {
 2432     return false;
 2433   }
 2434   int bit_size = vlen * type2aelembytes(bt) * 8;
 2435   if (UseSVE == 0 && bit_size > 128) {
 2436     return false;
 2437   }
 2438   if (UseSVE > 0) {
 2439     return op_sve_supported(opcode, vlen, bt);
 2440   } else { // NEON
 2441     // Special cases
 2442     switch (opcode) {
 2443     case Op_VectorMaskCmp:
 2444       if (vlen < 2 || bit_size < 64) {
 2445         return false;
 2446       }
 2447       break;
 2448     case Op_MulAddVS2VI:
 2449       if (bit_size < 128) {
 2450         return false;
 2451       }
 2452       break;
 2453     case Op_MulVL:
 2454       return false;
 2455     case Op_VectorLoadShuffle:
 2456     case Op_VectorRearrange:
 2457       if (vlen < 4) {
 2458         return false;
 2459       }
 2460       break;
 2461     case Op_LoadVectorGather:
 2462     case Op_StoreVectorScatter:
 2463       return false;
 2464     default:
 2465       break;
 2466     }
 2467   }
 2468   return vector_size_supported(bt, vlen);
 2469 }
 2470 
 2471 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
 2472   // Only SVE supports masked operations.
 2473   if (UseSVE == 0) {
 2474     return false;
 2475   }
 2476   return match_rule_supported(opcode) &&
 2477          masked_op_sve_supported(opcode, vlen, bt);
 2478 }
 2479 
 2480 const RegMask* Matcher::predicate_reg_mask(void) {
 2481   return &_PR_REG_mask;
 2482 }
 2483 
 2484 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2485   return new TypeVectMask(elemTy, length);
 2486 }
 2487 
 2488 // Vector calling convention not yet implemented.
 2489 const bool Matcher::supports_vector_calling_convention(void) {
 2490   return false;
 2491 }
 2492 
 2493 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2494   Unimplemented();
 2495   return OptoRegPair(0, 0);
 2496 }
 2497 
 2498 // Is this branch offset short enough that a short branch can be used?
 2499 //
 2500 // NOTE: If the platform does not provide any short branch variants, then
 2501 //       this method should return false for offset 0.
 2502 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2503   // The passed offset is relative to address of the branch.
 2504 
 2505   return (-32768 <= offset && offset < 32768);
 2506 }
 2507 
 2508 // Vector width in bytes.
 2509 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2510   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2511   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2512   // Minimum 2 values in vector
 2513   if (size < 2*type2aelembytes(bt)) size = 0;
 2514   // But never < 4
 2515   if (size < 4) size = 0;
 2516   return size;
 2517 }
 2518 
 2519 // Limits on vector size (number of elements) loaded into vector.
 2520 const int Matcher::max_vector_size(const BasicType bt) {
 2521   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2522 }
 2523 
 2524 const int Matcher::min_vector_size(const BasicType bt) {
 2525   int max_size = max_vector_size(bt);
 2526   // Limit the min vector size to 8 bytes.
 2527   int size = 8 / type2aelembytes(bt);
 2528   if (bt == T_BYTE) {
 2529     // To support vector api shuffle/rearrange.
 2530     size = 4;
 2531   } else if (bt == T_BOOLEAN) {
 2532     // To support vector api load/store mask.
 2533     size = 2;
 2534   }
 2535   if (size < 2) size = 2;
 2536   return MIN2(size, max_size);
 2537 }
 2538 
 2539 // Actual max scalable vector register length.
 2540 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2541   return Matcher::max_vector_size(bt);
 2542 }
 2543 
 2544 // Vector ideal reg.
 2545 const uint Matcher::vector_ideal_reg(int len) {
 2546   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2547     return Op_VecA;
 2548   }
 2549   switch(len) {
 2550     // For 16-bit/32-bit mask vector, reuse VecD.
 2551     case  2:
 2552     case  4:
 2553     case  8: return Op_VecD;
 2554     case 16: return Op_VecX;
 2555   }
 2556   ShouldNotReachHere();
 2557   return 0;
 2558 }
 2559 
 2560 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2561   ShouldNotReachHere(); // generic vector operands not supported
 2562   return NULL;
 2563 }
 2564 
 2565 bool Matcher::is_reg2reg_move(MachNode* m) {
 2566   ShouldNotReachHere();  // generic vector operands not supported
 2567   return false;
 2568 }
 2569 
 2570 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2571   ShouldNotReachHere();  // generic vector operands not supported
 2572   return false;
 2573 }
 2574 
 2575 // Return whether or not this register is ever used as an argument.
 2576 // This function is used on startup to build the trampoline stubs in
 2577 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2578 // call in the trampoline, and arguments in those registers not be
 2579 // available to the callee.
 2580 bool Matcher::can_be_java_arg(int reg)
 2581 {
 2582   return
 2583     reg ==  R0_num || reg == R0_H_num ||
 2584     reg ==  R1_num || reg == R1_H_num ||
 2585     reg ==  R2_num || reg == R2_H_num ||
 2586     reg ==  R3_num || reg == R3_H_num ||
 2587     reg ==  R4_num || reg == R4_H_num ||
 2588     reg ==  R5_num || reg == R5_H_num ||
 2589     reg ==  R6_num || reg == R6_H_num ||
 2590     reg ==  R7_num || reg == R7_H_num ||
 2591     reg ==  V0_num || reg == V0_H_num ||
 2592     reg ==  V1_num || reg == V1_H_num ||
 2593     reg ==  V2_num || reg == V2_H_num ||
 2594     reg ==  V3_num || reg == V3_H_num ||
 2595     reg ==  V4_num || reg == V4_H_num ||
 2596     reg ==  V5_num || reg == V5_H_num ||
 2597     reg ==  V6_num || reg == V6_H_num ||
 2598     reg ==  V7_num || reg == V7_H_num;
 2599 }
 2600 
 2601 bool Matcher::is_spillable_arg(int reg)
 2602 {
 2603   return can_be_java_arg(reg);
 2604 }
 2605 
 2606 uint Matcher::int_pressure_limit()
 2607 {
 2608   // JDK-8183543: When taking the number of available registers as int
 2609   // register pressure threshold, the jtreg test:
 2610   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2611   // failed due to C2 compilation failure with
 2612   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2613   //
 2614   // A derived pointer is live at CallNode and then is flagged by RA
 2615   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2616   // derived pointers and lastly fail to spill after reaching maximum
 2617   // number of iterations. Lowering the default pressure threshold to
 2618   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2619   // a high register pressure area of the code so that split_DEF can
 2620   // generate DefinitionSpillCopy for the derived pointer.
 2621   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2622   if (!PreserveFramePointer) {
 2623     // When PreserveFramePointer is off, frame pointer is allocatable,
 2624     // but different from other SOC registers, it is excluded from
 2625     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2626     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2627     // See check_pressure_at_fatproj().
 2628     default_int_pressure_threshold--;
 2629   }
 2630   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2631 }
 2632 
 2633 uint Matcher::float_pressure_limit()
 2634 {
 2635   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2636   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2637 }
 2638 
 2639 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2640   return false;
 2641 }
 2642 
 2643 RegMask Matcher::divI_proj_mask() {
 2644   ShouldNotReachHere();
 2645   return RegMask();
 2646 }
 2647 
 2648 // Register for MODI projection of divmodI.
 2649 RegMask Matcher::modI_proj_mask() {
 2650   ShouldNotReachHere();
 2651   return RegMask();
 2652 }
 2653 
 2654 // Register for DIVL projection of divmodL.
 2655 RegMask Matcher::divL_proj_mask() {
 2656   ShouldNotReachHere();
 2657   return RegMask();
 2658 }
 2659 
 2660 // Register for MODL projection of divmodL.
 2661 RegMask Matcher::modL_proj_mask() {
 2662   ShouldNotReachHere();
 2663   return RegMask();
 2664 }
 2665 
 2666 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2667   return FP_REG_mask();
 2668 }
 2669 
 2670 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2671   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2672     Node* u = addp->fast_out(i);
 2673     if (u->is_LoadStore()) {
 2674       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2675       // instructions) only take register indirect as an operand, so
 2676       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2677       // must fail.
 2678       return false;
 2679     }
 2680     if (u->is_Mem()) {
 2681       int opsize = u->as_Mem()->memory_size();
 2682       assert(opsize > 0, "unexpected memory operand size");
 2683       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2684         return false;
 2685       }
 2686     }
 2687   }
 2688   return true;
 2689 }
 2690 
 2691 bool can_combine_with_imm(Node* binary_node, Node* replicate_node) {
 2692   if (UseSVE == 0 || !VectorNode::is_invariant_vector(replicate_node)){
 2693     return false;
 2694   }
 2695   Node* imm_node = replicate_node->in(1);
 2696   if (!imm_node->is_Con()) {
 2697     return false;
 2698   }
 2699 
 2700   const Type* t = imm_node->bottom_type();
 2701   if (!(t->isa_int() || t->isa_long())) {
 2702     return false;
 2703   }
 2704 
 2705   switch (binary_node->Opcode()) {
 2706   case Op_AndV:
 2707   case Op_OrV:
 2708   case Op_XorV: {
 2709     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(binary_node));
 2710     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2711     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2712   }
 2713   case Op_AddVB:
 2714     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2715   case Op_AddVS:
 2716   case Op_AddVI:
 2717     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2718   case Op_AddVL:
 2719     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2720   default:
 2721     return false;
 2722   }
 2723 }
 2724 
 2725 bool is_vector_arith_imm_pattern(Node* n, Node* m) {
 2726   if (n != NULL && m != NULL) {
 2727     return can_combine_with_imm(n, m);
 2728   }
 2729   return false;
 2730 }
 2731 
 2732 // Should the matcher clone input 'm' of node 'n'?
 2733 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2734   // ShiftV src (ShiftCntV con)
 2735   // StoreVector (VectorStoreMask src)
 2736   // Binary src (Replicate con)
 2737   if (is_vshift_con_pattern(n, m) ||
 2738       (UseSVE > 0 && m->Opcode() == Op_VectorStoreMask && n->Opcode() == Op_StoreVector) ||
 2739       is_vector_arith_imm_pattern(n, m)) {
 2740     mstack.push(m, Visit);
 2741     return true;
 2742   }
 2743 
 2744   return false;
 2745 }
 2746 
 2747 // Should the Matcher clone shifts on addressing modes, expecting them
 2748 // to be subsumed into complex addressing expressions or compute them
 2749 // into registers?
 2750 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2751   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2752     return true;
 2753   }
 2754 
 2755   Node *off = m->in(AddPNode::Offset);
 2756   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2757       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2758       // Are there other uses besides address expressions?
 2759       !is_visited(off)) {
 2760     address_visited.set(off->_idx); // Flag as address_visited
 2761     mstack.push(off->in(2), Visit);
 2762     Node *conv = off->in(1);
 2763     if (conv->Opcode() == Op_ConvI2L &&
 2764         // Are there other uses besides address expressions?
 2765         !is_visited(conv)) {
 2766       address_visited.set(conv->_idx); // Flag as address_visited
 2767       mstack.push(conv->in(1), Pre_Visit);
 2768     } else {
 2769       mstack.push(conv, Pre_Visit);
 2770     }
 2771     address_visited.test_set(m->_idx); // Flag as address_visited
 2772     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2773     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2774     return true;
 2775   } else if (off->Opcode() == Op_ConvI2L &&
 2776              // Are there other uses besides address expressions?
 2777              !is_visited(off)) {
 2778     address_visited.test_set(m->_idx); // Flag as address_visited
 2779     address_visited.set(off->_idx); // Flag as address_visited
 2780     mstack.push(off->in(1), Pre_Visit);
 2781     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2782     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2783     return true;
 2784   }
 2785   return false;
 2786 }
 2787 
 2788 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2789   C2_MacroAssembler _masm(&cbuf);                                       \
 2790   {                                                                     \
 2791     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2792     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2793     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2794     __ INSN(REG, as_Register(BASE));                                    \
 2795   }
 2796 
 2797 
 2798 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2799   {
 2800     Address::extend scale;
 2801 
 2802     // Hooboy, this is fugly.  We need a way to communicate to the
 2803     // encoder that the index needs to be sign extended, so we have to
 2804     // enumerate all the cases.
 2805     switch (opcode) {
 2806     case INDINDEXSCALEDI2L:
 2807     case INDINDEXSCALEDI2LN:
 2808     case INDINDEXI2L:
 2809     case INDINDEXI2LN:
 2810       scale = Address::sxtw(size);
 2811       break;
 2812     default:
 2813       scale = Address::lsl(size);
 2814     }
 2815 
 2816     if (index == -1) {
 2817       return Address(base, disp);
 2818     } else {
 2819       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2820       return Address(base, as_Register(index), scale);
 2821     }
 2822   }
 2823 
 2824 
 2825 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2826 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2827 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2828 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2829                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2830 
 2831   // Used for all non-volatile memory accesses.  The use of
 2832   // $mem->opcode() to discover whether this pattern uses sign-extended
 2833   // offsets is something of a kludge.
 2834   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2835                         Register reg, int opcode,
 2836                         Register base, int index, int scale, int disp,
 2837                         int size_in_memory)
 2838   {
 2839     Address addr = mem2address(opcode, base, index, scale, disp);
 2840     if (addr.getMode() == Address::base_plus_offset) {
 2841       /* If we get an out-of-range offset it is a bug in the compiler,
 2842          so we assert here. */
 2843       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2844              "c2 compiler bug");
 2845       /* Fix up any out-of-range offsets. */
 2846       assert_different_registers(rscratch1, base);
 2847       assert_different_registers(rscratch1, reg);
 2848       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2849     }
 2850     (masm.*insn)(reg, addr);
 2851   }
 2852 
 2853   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2854                         FloatRegister reg, int opcode,
 2855                         Register base, int index, int size, int disp,
 2856                         int size_in_memory)
 2857   {
 2858     Address::extend scale;
 2859 
 2860     switch (opcode) {
 2861     case INDINDEXSCALEDI2L:
 2862     case INDINDEXSCALEDI2LN:
 2863       scale = Address::sxtw(size);
 2864       break;
 2865     default:
 2866       scale = Address::lsl(size);
 2867     }
 2868 
 2869     if (index == -1) {
 2870       /* If we get an out-of-range offset it is a bug in the compiler,
 2871          so we assert here. */
 2872       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2873       /* Fix up any out-of-range offsets. */
 2874       assert_different_registers(rscratch1, base);
 2875       Address addr = Address(base, disp);
 2876       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2877       (masm.*insn)(reg, addr);
 2878     } else {
 2879       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2880       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2881     }
 2882   }
 2883 
 2884   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2885                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2886                         int opcode, Register base, int index, int size, int disp)
 2887   {
 2888     if (index == -1) {
 2889       (masm.*insn)(reg, T, Address(base, disp));
 2890     } else {
 2891       assert(disp == 0, "unsupported address mode");
 2892       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2893     }
 2894   }
 2895 
 2896 %}
 2897 
 2898 
 2899 
 2900 //----------ENCODING BLOCK-----------------------------------------------------
 2901 // This block specifies the encoding classes used by the compiler to
 2902 // output byte streams.  Encoding classes are parameterized macros
 2903 // used by Machine Instruction Nodes in order to generate the bit
 2904 // encoding of the instruction.  Operands specify their base encoding
 2905 // interface with the interface keyword.  There are currently
 2906 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2907 // COND_INTER.  REG_INTER causes an operand to generate a function
 2908 // which returns its register number when queried.  CONST_INTER causes
 2909 // an operand to generate a function which returns the value of the
 2910 // constant when queried.  MEMORY_INTER causes an operand to generate
 2911 // four functions which return the Base Register, the Index Register,
 2912 // the Scale Value, and the Offset Value of the operand when queried.
 2913 // COND_INTER causes an operand to generate six functions which return
 2914 // the encoding code (ie - encoding bits for the instruction)
 2915 // associated with each basic boolean condition for a conditional
 2916 // instruction.
 2917 //
 2918 // Instructions specify two basic values for encoding.  Again, a
 2919 // function is available to check if the constant displacement is an
 2920 // oop. They use the ins_encode keyword to specify their encoding
 2921 // classes (which must be a sequence of enc_class names, and their
 2922 // parameters, specified in the encoding block), and they use the
 2923 // opcode keyword to specify, in order, their primary, secondary, and
 2924 // tertiary opcode.  Only the opcode sections which a particular
 2925 // instruction needs for encoding need to be specified.
 2926 encode %{
 2927   // Build emit functions for each basic byte or larger field in the
 2928   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2929   // from C++ code in the enc_class source block.  Emit functions will
 2930   // live in the main source block for now.  In future, we can
 2931   // generalize this by adding a syntax that specifies the sizes of
 2932   // fields in an order, so that the adlc can build the emit functions
 2933   // automagically
 2934 
 2935   // catch all for unimplemented encodings
 2936   enc_class enc_unimplemented %{
 2937     C2_MacroAssembler _masm(&cbuf);
 2938     __ unimplemented("C2 catch all");
 2939   %}
 2940 
 2941   // BEGIN Non-volatile memory access
 2942 
 2943   // This encoding class is generated automatically from ad_encode.m4.
 2944   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2945   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2946     Register dst_reg = as_Register($dst$$reg);
 2947     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2948                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2949   %}
 2950 
 2951   // This encoding class is generated automatically from ad_encode.m4.
 2952   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2953   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2954     Register dst_reg = as_Register($dst$$reg);
 2955     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2956                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2957   %}
 2958 
 2959   // This encoding class is generated automatically from ad_encode.m4.
 2960   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2961   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2962     Register dst_reg = as_Register($dst$$reg);
 2963     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2964                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2965   %}
 2966 
 2967   // This encoding class is generated automatically from ad_encode.m4.
 2968   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2969   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2970     Register dst_reg = as_Register($dst$$reg);
 2971     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2972                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2973   %}
 2974 
 2975   // This encoding class is generated automatically from ad_encode.m4.
 2976   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2977   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2978     Register dst_reg = as_Register($dst$$reg);
 2979     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2980                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2981   %}
 2982 
 2983   // This encoding class is generated automatically from ad_encode.m4.
 2984   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2985   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2986     Register dst_reg = as_Register($dst$$reg);
 2987     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2988                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2989   %}
 2990 
 2991   // This encoding class is generated automatically from ad_encode.m4.
 2992   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2993   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2994     Register dst_reg = as_Register($dst$$reg);
 2995     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2996                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2997   %}
 2998 
 2999   // This encoding class is generated automatically from ad_encode.m4.
 3000   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3001   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 3002     Register dst_reg = as_Register($dst$$reg);
 3003     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3004                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3005   %}
 3006 
 3007   // This encoding class is generated automatically from ad_encode.m4.
 3008   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3009   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 3010     Register dst_reg = as_Register($dst$$reg);
 3011     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3012                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3013   %}
 3014 
 3015   // This encoding class is generated automatically from ad_encode.m4.
 3016   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3017   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 3018     Register dst_reg = as_Register($dst$$reg);
 3019     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3020                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3021   %}
 3022 
 3023   // This encoding class is generated automatically from ad_encode.m4.
 3024   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3025   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 3026     Register dst_reg = as_Register($dst$$reg);
 3027     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 3028                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3029   %}
 3030 
 3031   // This encoding class is generated automatically from ad_encode.m4.
 3032   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3033   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 3034     Register dst_reg = as_Register($dst$$reg);
 3035     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 3036                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3037   %}
 3038 
 3039   // This encoding class is generated automatically from ad_encode.m4.
 3040   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3041   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3042     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3043     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3044                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3045   %}
 3046 
 3047   // This encoding class is generated automatically from ad_encode.m4.
 3048   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3049   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3050     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3051     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3052                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3053   %}
 3054 
 3055   // This encoding class is generated automatically from ad_encode.m4.
 3056   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3057   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3058     Register src_reg = as_Register($src$$reg);
 3059     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 3060                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3061   %}
 3062 
 3063   // This encoding class is generated automatically from ad_encode.m4.
 3064   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3065   enc_class aarch64_enc_strb0(memory1 mem) %{
 3066     C2_MacroAssembler _masm(&cbuf);
 3067     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3068                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3069   %}
 3070 
 3071   // This encoding class is generated automatically from ad_encode.m4.
 3072   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3073   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3074     Register src_reg = as_Register($src$$reg);
 3075     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3076                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3077   %}
 3078 
 3079   // This encoding class is generated automatically from ad_encode.m4.
 3080   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3081   enc_class aarch64_enc_strh0(memory2 mem) %{
 3082     C2_MacroAssembler _masm(&cbuf);
 3083     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3084                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3085   %}
 3086 
 3087   // This encoding class is generated automatically from ad_encode.m4.
 3088   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3089   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3090     Register src_reg = as_Register($src$$reg);
 3091     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3092                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3093   %}
 3094 
 3095   // This encoding class is generated automatically from ad_encode.m4.
 3096   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3097   enc_class aarch64_enc_strw0(memory4 mem) %{
 3098     C2_MacroAssembler _masm(&cbuf);
 3099     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3100                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3101   %}
 3102 
 3103   // This encoding class is generated automatically from ad_encode.m4.
 3104   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3105   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3106     Register src_reg = as_Register($src$$reg);
 3107     // we sometimes get asked to store the stack pointer into the
 3108     // current thread -- we cannot do that directly on AArch64
 3109     if (src_reg == r31_sp) {
 3110       C2_MacroAssembler _masm(&cbuf);
 3111       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3112       __ mov(rscratch2, sp);
 3113       src_reg = rscratch2;
 3114     }
 3115     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3116                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3117   %}
 3118 
 3119   // This encoding class is generated automatically from ad_encode.m4.
 3120   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3121   enc_class aarch64_enc_str0(memory8 mem) %{
 3122     C2_MacroAssembler _masm(&cbuf);
 3123     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3124                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3125   %}
 3126 
 3127   // This encoding class is generated automatically from ad_encode.m4.
 3128   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3129   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3130     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3131     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3132                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3133   %}
 3134 
 3135   // This encoding class is generated automatically from ad_encode.m4.
 3136   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3137   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3138     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3139     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3140                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3141   %}
 3142 
 3143   // This encoding class is generated automatically from ad_encode.m4.
 3144   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3145   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3146       C2_MacroAssembler _masm(&cbuf);
 3147       __ membar(Assembler::StoreStore);
 3148       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3149                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3150   %}
 3151 
 3152   // END Non-volatile memory access
 3153 
 3154   // Vector loads and stores
 3155   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3156     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3157     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3158        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3159   %}
 3160 
 3161   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3162     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3163     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3164        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3165   %}
 3166 
 3167   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3168     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3169     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3170        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3171   %}
 3172 
 3173   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3174     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3175     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3176        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3177   %}
 3178 
 3179   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3180     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3181     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3182        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3183   %}
 3184 
 3185   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3186     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3187     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3188        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3189   %}
 3190 
 3191   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3192     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3193     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3194        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3195   %}
 3196 
 3197   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3198     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3199     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3200        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3201   %}
 3202 
 3203   // volatile loads and stores
 3204 
 3205   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3206     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3207                  rscratch1, stlrb);
 3208   %}
 3209 
 3210   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3211     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3212                  rscratch1, stlrh);
 3213   %}
 3214 
 3215   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3216     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3217                  rscratch1, stlrw);
 3218   %}
 3219 
 3220 
 3221   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3222     Register dst_reg = as_Register($dst$$reg);
 3223     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3224              rscratch1, ldarb);
 3225     __ sxtbw(dst_reg, dst_reg);
 3226   %}
 3227 
 3228   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3229     Register dst_reg = as_Register($dst$$reg);
 3230     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3231              rscratch1, ldarb);
 3232     __ sxtb(dst_reg, dst_reg);
 3233   %}
 3234 
 3235   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3236     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3237              rscratch1, ldarb);
 3238   %}
 3239 
 3240   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3241     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3242              rscratch1, ldarb);
 3243   %}
 3244 
 3245   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3246     Register dst_reg = as_Register($dst$$reg);
 3247     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3248              rscratch1, ldarh);
 3249     __ sxthw(dst_reg, dst_reg);
 3250   %}
 3251 
 3252   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3253     Register dst_reg = as_Register($dst$$reg);
 3254     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3255              rscratch1, ldarh);
 3256     __ sxth(dst_reg, dst_reg);
 3257   %}
 3258 
 3259   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3260     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3261              rscratch1, ldarh);
 3262   %}
 3263 
 3264   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3265     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3266              rscratch1, ldarh);
 3267   %}
 3268 
 3269   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3270     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3271              rscratch1, ldarw);
 3272   %}
 3273 
 3274   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3275     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3276              rscratch1, ldarw);
 3277   %}
 3278 
 3279   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3280     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3281              rscratch1, ldar);
 3282   %}
 3283 
 3284   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3285     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3286              rscratch1, ldarw);
 3287     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3288   %}
 3289 
 3290   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3291     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3292              rscratch1, ldar);
 3293     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3294   %}
 3295 
 3296   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3297     Register src_reg = as_Register($src$$reg);
 3298     // we sometimes get asked to store the stack pointer into the
 3299     // current thread -- we cannot do that directly on AArch64
 3300     if (src_reg == r31_sp) {
 3301       C2_MacroAssembler _masm(&cbuf);
 3302       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3303       __ mov(rscratch2, sp);
 3304       src_reg = rscratch2;
 3305     }
 3306     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3307                  rscratch1, stlr);
 3308   %}
 3309 
 3310   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3311     {
 3312       C2_MacroAssembler _masm(&cbuf);
 3313       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3314       __ fmovs(rscratch2, src_reg);
 3315     }
 3316     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3317                  rscratch1, stlrw);
 3318   %}
 3319 
 3320   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3321     {
 3322       C2_MacroAssembler _masm(&cbuf);
 3323       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3324       __ fmovd(rscratch2, src_reg);
 3325     }
 3326     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3327                  rscratch1, stlr);
 3328   %}
 3329 
 3330   // synchronized read/update encodings
 3331 
 3332   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3333     C2_MacroAssembler _masm(&cbuf);
 3334     Register dst_reg = as_Register($dst$$reg);
 3335     Register base = as_Register($mem$$base);
 3336     int index = $mem$$index;
 3337     int scale = $mem$$scale;
 3338     int disp = $mem$$disp;
 3339     if (index == -1) {
 3340        if (disp != 0) {
 3341         __ lea(rscratch1, Address(base, disp));
 3342         __ ldaxr(dst_reg, rscratch1);
 3343       } else {
 3344         // TODO
 3345         // should we ever get anything other than this case?
 3346         __ ldaxr(dst_reg, base);
 3347       }
 3348     } else {
 3349       Register index_reg = as_Register(index);
 3350       if (disp == 0) {
 3351         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3352         __ ldaxr(dst_reg, rscratch1);
 3353       } else {
 3354         __ lea(rscratch1, Address(base, disp));
 3355         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3356         __ ldaxr(dst_reg, rscratch1);
 3357       }
 3358     }
 3359   %}
 3360 
 3361   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3362     C2_MacroAssembler _masm(&cbuf);
 3363     Register src_reg = as_Register($src$$reg);
 3364     Register base = as_Register($mem$$base);
 3365     int index = $mem$$index;
 3366     int scale = $mem$$scale;
 3367     int disp = $mem$$disp;
 3368     if (index == -1) {
 3369        if (disp != 0) {
 3370         __ lea(rscratch2, Address(base, disp));
 3371         __ stlxr(rscratch1, src_reg, rscratch2);
 3372       } else {
 3373         // TODO
 3374         // should we ever get anything other than this case?
 3375         __ stlxr(rscratch1, src_reg, base);
 3376       }
 3377     } else {
 3378       Register index_reg = as_Register(index);
 3379       if (disp == 0) {
 3380         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3381         __ stlxr(rscratch1, src_reg, rscratch2);
 3382       } else {
 3383         __ lea(rscratch2, Address(base, disp));
 3384         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3385         __ stlxr(rscratch1, src_reg, rscratch2);
 3386       }
 3387     }
 3388     __ cmpw(rscratch1, zr);
 3389   %}
 3390 
 3391   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3392     C2_MacroAssembler _masm(&cbuf);
 3393     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3394     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3395                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3396                /*weak*/ false, noreg);
 3397   %}
 3398 
 3399   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3400     C2_MacroAssembler _masm(&cbuf);
 3401     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3402     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3403                Assembler::word, /*acquire*/ false, /*release*/ true,
 3404                /*weak*/ false, noreg);
 3405   %}
 3406 
 3407   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3408     C2_MacroAssembler _masm(&cbuf);
 3409     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3410     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3411                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3412                /*weak*/ false, noreg);
 3413   %}
 3414 
 3415   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3416     C2_MacroAssembler _masm(&cbuf);
 3417     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3418     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3419                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3420                /*weak*/ false, noreg);
 3421   %}
 3422 
 3423 
 3424   // The only difference between aarch64_enc_cmpxchg and
 3425   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3426   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3427   // lock.
 3428   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3429     C2_MacroAssembler _masm(&cbuf);
 3430     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3431     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3432                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3433                /*weak*/ false, noreg);
 3434   %}
 3435 
 3436   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3437     C2_MacroAssembler _masm(&cbuf);
 3438     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3439     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3440                Assembler::word, /*acquire*/ true, /*release*/ true,
 3441                /*weak*/ false, noreg);
 3442   %}
 3443 
 3444   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3445     C2_MacroAssembler _masm(&cbuf);
 3446     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3447     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3448                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3449                /*weak*/ false, noreg);
 3450   %}
 3451 
 3452   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3453     C2_MacroAssembler _masm(&cbuf);
 3454     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3455     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3456                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3457                /*weak*/ false, noreg);
 3458   %}
 3459 
 3460   // auxiliary used for CompareAndSwapX to set result register
 3461   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3462     C2_MacroAssembler _masm(&cbuf);
 3463     Register res_reg = as_Register($res$$reg);
 3464     __ cset(res_reg, Assembler::EQ);
 3465   %}
 3466 
 3467   // prefetch encodings
 3468 
 3469   enc_class aarch64_enc_prefetchw(memory mem) %{
 3470     C2_MacroAssembler _masm(&cbuf);
 3471     Register base = as_Register($mem$$base);
 3472     int index = $mem$$index;
 3473     int scale = $mem$$scale;
 3474     int disp = $mem$$disp;
 3475     if (index == -1) {
 3476       __ prfm(Address(base, disp), PSTL1KEEP);
 3477     } else {
 3478       Register index_reg = as_Register(index);
 3479       if (disp == 0) {
 3480         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3481       } else {
 3482         __ lea(rscratch1, Address(base, disp));
 3483 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3484       }
 3485     }
 3486   %}
 3487 
 3488   /// mov envcodings
 3489 
 3490   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3491     C2_MacroAssembler _masm(&cbuf);
 3492     uint32_t con = (uint32_t)$src$$constant;
 3493     Register dst_reg = as_Register($dst$$reg);
 3494     if (con == 0) {
 3495       __ movw(dst_reg, zr);
 3496     } else {
 3497       __ movw(dst_reg, con);
 3498     }
 3499   %}
 3500 
 3501   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3502     C2_MacroAssembler _masm(&cbuf);
 3503     Register dst_reg = as_Register($dst$$reg);
 3504     uint64_t con = (uint64_t)$src$$constant;
 3505     if (con == 0) {
 3506       __ mov(dst_reg, zr);
 3507     } else {
 3508       __ mov(dst_reg, con);
 3509     }
 3510   %}
 3511 
 3512   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3513     C2_MacroAssembler _masm(&cbuf);
 3514     Register dst_reg = as_Register($dst$$reg);
 3515     address con = (address)$src$$constant;
 3516     if (con == NULL || con == (address)1) {
 3517       ShouldNotReachHere();
 3518     } else {
 3519       relocInfo::relocType rtype = $src->constant_reloc();
 3520       if (rtype == relocInfo::oop_type) {
 3521         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3522       } else if (rtype == relocInfo::metadata_type) {
 3523         __ mov_metadata(dst_reg, (Metadata*)con);
 3524       } else {
 3525         assert(rtype == relocInfo::none, "unexpected reloc type");
 3526         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3527           __ mov(dst_reg, con);
 3528         } else {
 3529           uint64_t offset;
 3530           __ adrp(dst_reg, con, offset);
 3531           __ add(dst_reg, dst_reg, offset);
 3532         }
 3533       }
 3534     }
 3535   %}
 3536 
 3537   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3538     C2_MacroAssembler _masm(&cbuf);
 3539     Register dst_reg = as_Register($dst$$reg);
 3540     __ mov(dst_reg, zr);
 3541   %}
 3542 
 3543   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3544     C2_MacroAssembler _masm(&cbuf);
 3545     Register dst_reg = as_Register($dst$$reg);
 3546     __ mov(dst_reg, (uint64_t)1);
 3547   %}
 3548 
 3549   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3550     C2_MacroAssembler _masm(&cbuf);
 3551     __ load_byte_map_base($dst$$Register);
 3552   %}
 3553 
 3554   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3555     C2_MacroAssembler _masm(&cbuf);
 3556     Register dst_reg = as_Register($dst$$reg);
 3557     address con = (address)$src$$constant;
 3558     if (con == NULL) {
 3559       ShouldNotReachHere();
 3560     } else {
 3561       relocInfo::relocType rtype = $src->constant_reloc();
 3562       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3563       __ set_narrow_oop(dst_reg, (jobject)con);
 3564     }
 3565   %}
 3566 
 3567   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3568     C2_MacroAssembler _masm(&cbuf);
 3569     Register dst_reg = as_Register($dst$$reg);
 3570     __ mov(dst_reg, zr);
 3571   %}
 3572 
 3573   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3574     C2_MacroAssembler _masm(&cbuf);
 3575     Register dst_reg = as_Register($dst$$reg);
 3576     address con = (address)$src$$constant;
 3577     if (con == NULL) {
 3578       ShouldNotReachHere();
 3579     } else {
 3580       relocInfo::relocType rtype = $src->constant_reloc();
 3581       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3582       __ set_narrow_klass(dst_reg, (Klass *)con);
 3583     }
 3584   %}
 3585 
 3586   // arithmetic encodings
 3587 
 3588   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3589     C2_MacroAssembler _masm(&cbuf);
 3590     Register dst_reg = as_Register($dst$$reg);
 3591     Register src_reg = as_Register($src1$$reg);
 3592     int32_t con = (int32_t)$src2$$constant;
 3593     // add has primary == 0, subtract has primary == 1
 3594     if ($primary) { con = -con; }
 3595     if (con < 0) {
 3596       __ subw(dst_reg, src_reg, -con);
 3597     } else {
 3598       __ addw(dst_reg, src_reg, con);
 3599     }
 3600   %}
 3601 
 3602   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3603     C2_MacroAssembler _masm(&cbuf);
 3604     Register dst_reg = as_Register($dst$$reg);
 3605     Register src_reg = as_Register($src1$$reg);
 3606     int32_t con = (int32_t)$src2$$constant;
 3607     // add has primary == 0, subtract has primary == 1
 3608     if ($primary) { con = -con; }
 3609     if (con < 0) {
 3610       __ sub(dst_reg, src_reg, -con);
 3611     } else {
 3612       __ add(dst_reg, src_reg, con);
 3613     }
 3614   %}
 3615 
 3616   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3617     C2_MacroAssembler _masm(&cbuf);
 3618    Register dst_reg = as_Register($dst$$reg);
 3619    Register src1_reg = as_Register($src1$$reg);
 3620    Register src2_reg = as_Register($src2$$reg);
 3621     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3622   %}
 3623 
 3624   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3625     C2_MacroAssembler _masm(&cbuf);
 3626    Register dst_reg = as_Register($dst$$reg);
 3627    Register src1_reg = as_Register($src1$$reg);
 3628    Register src2_reg = as_Register($src2$$reg);
 3629     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3630   %}
 3631 
 3632   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3633     C2_MacroAssembler _masm(&cbuf);
 3634    Register dst_reg = as_Register($dst$$reg);
 3635    Register src1_reg = as_Register($src1$$reg);
 3636    Register src2_reg = as_Register($src2$$reg);
 3637     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3638   %}
 3639 
 3640   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3641     C2_MacroAssembler _masm(&cbuf);
 3642    Register dst_reg = as_Register($dst$$reg);
 3643    Register src1_reg = as_Register($src1$$reg);
 3644    Register src2_reg = as_Register($src2$$reg);
 3645     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3646   %}
 3647 
 3648   // compare instruction encodings
 3649 
 3650   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3651     C2_MacroAssembler _masm(&cbuf);
 3652     Register reg1 = as_Register($src1$$reg);
 3653     Register reg2 = as_Register($src2$$reg);
 3654     __ cmpw(reg1, reg2);
 3655   %}
 3656 
 3657   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3658     C2_MacroAssembler _masm(&cbuf);
 3659     Register reg = as_Register($src1$$reg);
 3660     int32_t val = $src2$$constant;
 3661     if (val >= 0) {
 3662       __ subsw(zr, reg, val);
 3663     } else {
 3664       __ addsw(zr, reg, -val);
 3665     }
 3666   %}
 3667 
 3668   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3669     C2_MacroAssembler _masm(&cbuf);
 3670     Register reg1 = as_Register($src1$$reg);
 3671     uint32_t val = (uint32_t)$src2$$constant;
 3672     __ movw(rscratch1, val);
 3673     __ cmpw(reg1, rscratch1);
 3674   %}
 3675 
 3676   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3677     C2_MacroAssembler _masm(&cbuf);
 3678     Register reg1 = as_Register($src1$$reg);
 3679     Register reg2 = as_Register($src2$$reg);
 3680     __ cmp(reg1, reg2);
 3681   %}
 3682 
 3683   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3684     C2_MacroAssembler _masm(&cbuf);
 3685     Register reg = as_Register($src1$$reg);
 3686     int64_t val = $src2$$constant;
 3687     if (val >= 0) {
 3688       __ subs(zr, reg, val);
 3689     } else if (val != -val) {
 3690       __ adds(zr, reg, -val);
 3691     } else {
 3692     // aargh, Long.MIN_VALUE is a special case
 3693       __ orr(rscratch1, zr, (uint64_t)val);
 3694       __ subs(zr, reg, rscratch1);
 3695     }
 3696   %}
 3697 
 3698   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3699     C2_MacroAssembler _masm(&cbuf);
 3700     Register reg1 = as_Register($src1$$reg);
 3701     uint64_t val = (uint64_t)$src2$$constant;
 3702     __ mov(rscratch1, val);
 3703     __ cmp(reg1, rscratch1);
 3704   %}
 3705 
 3706   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3707     C2_MacroAssembler _masm(&cbuf);
 3708     Register reg1 = as_Register($src1$$reg);
 3709     Register reg2 = as_Register($src2$$reg);
 3710     __ cmp(reg1, reg2);
 3711   %}
 3712 
 3713   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3714     C2_MacroAssembler _masm(&cbuf);
 3715     Register reg1 = as_Register($src1$$reg);
 3716     Register reg2 = as_Register($src2$$reg);
 3717     __ cmpw(reg1, reg2);
 3718   %}
 3719 
 3720   enc_class aarch64_enc_testp(iRegP src) %{
 3721     C2_MacroAssembler _masm(&cbuf);
 3722     Register reg = as_Register($src$$reg);
 3723     __ cmp(reg, zr);
 3724   %}
 3725 
 3726   enc_class aarch64_enc_testn(iRegN src) %{
 3727     C2_MacroAssembler _masm(&cbuf);
 3728     Register reg = as_Register($src$$reg);
 3729     __ cmpw(reg, zr);
 3730   %}
 3731 
 3732   enc_class aarch64_enc_b(label lbl) %{
 3733     C2_MacroAssembler _masm(&cbuf);
 3734     Label *L = $lbl$$label;
 3735     __ b(*L);
 3736   %}
 3737 
 3738   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3739     C2_MacroAssembler _masm(&cbuf);
 3740     Label *L = $lbl$$label;
 3741     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3742   %}
 3743 
 3744   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3745     C2_MacroAssembler _masm(&cbuf);
 3746     Label *L = $lbl$$label;
 3747     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3748   %}
 3749 
 3750   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3751   %{
 3752      Register sub_reg = as_Register($sub$$reg);
 3753      Register super_reg = as_Register($super$$reg);
 3754      Register temp_reg = as_Register($temp$$reg);
 3755      Register result_reg = as_Register($result$$reg);
 3756 
 3757      Label miss;
 3758      C2_MacroAssembler _masm(&cbuf);
 3759      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3760                                      NULL, &miss,
 3761                                      /*set_cond_codes:*/ true);
 3762      if ($primary) {
 3763        __ mov(result_reg, zr);
 3764      }
 3765      __ bind(miss);
 3766   %}
 3767 
 3768   enc_class aarch64_enc_java_static_call(method meth) %{
 3769     C2_MacroAssembler _masm(&cbuf);
 3770 
 3771     address addr = (address)$meth$$method;
 3772     address call;
 3773     if (!_method) {
 3774       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3775       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3776       if (call == NULL) {
 3777         ciEnv::current()->record_failure("CodeCache is full");
 3778         return;
 3779       }
 3780     } else {
 3781       int method_index = resolved_method_index(cbuf);
 3782       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3783                                                   : static_call_Relocation::spec(method_index);
 3784       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3785       if (call == NULL) {
 3786         ciEnv::current()->record_failure("CodeCache is full");
 3787         return;
 3788       }
 3789       // Emit stub for static call
 3790       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3791       if (stub == NULL) {
 3792         ciEnv::current()->record_failure("CodeCache is full");
 3793         return;
 3794       }
 3795     }
 3796 


 3797     // Only non uncommon_trap calls need to reinitialize ptrue.
 3798     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3799       __ reinitialize_ptrue();
 3800     }
 3801   %}
 3802 
 3803   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3804     C2_MacroAssembler _masm(&cbuf);
 3805     int method_index = resolved_method_index(cbuf);
 3806     address call = __ ic_call((address)$meth$$method, method_index);

 3807     if (call == NULL) {
 3808       ciEnv::current()->record_failure("CodeCache is full");
 3809       return;
 3810     } else if (Compile::current()->max_vector_size() > 0) {
 3811       __ reinitialize_ptrue();
 3812     }
 3813   %}
 3814 
 3815   enc_class aarch64_enc_call_epilog() %{
 3816     C2_MacroAssembler _masm(&cbuf);
 3817     if (VerifyStackAtCalls) {
 3818       // Check that stack depth is unchanged: find majik cookie on stack
 3819       __ call_Unimplemented();
 3820     }
 3821   %}
 3822 
 3823   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3824     C2_MacroAssembler _masm(&cbuf);
 3825 
 3826     // some calls to generated routines (arraycopy code) are scheduled
 3827     // by C2 as runtime calls. if so we can call them using a br (they
 3828     // will be in a reachable segment) otherwise we have to use a blr
 3829     // which loads the absolute address into a register.
 3830     address entry = (address)$meth$$method;
 3831     CodeBlob *cb = CodeCache::find_blob(entry);
 3832     if (cb) {
 3833       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));

 3834       if (call == NULL) {
 3835         ciEnv::current()->record_failure("CodeCache is full");
 3836         return;
 3837       }
 3838     } else {
 3839       Label retaddr;
 3840       __ adr(rscratch2, retaddr);
 3841       __ lea(rscratch1, RuntimeAddress(entry));
 3842       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3843       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3844       __ blr(rscratch1);
 3845       __ bind(retaddr);

 3846       __ add(sp, sp, 2 * wordSize);
 3847     }
 3848     if (Compile::current()->max_vector_size() > 0) {
 3849       __ reinitialize_ptrue();
 3850     }
 3851   %}
 3852 
 3853   enc_class aarch64_enc_rethrow() %{
 3854     C2_MacroAssembler _masm(&cbuf);
 3855     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3856   %}
 3857 
 3858   enc_class aarch64_enc_ret() %{
 3859     C2_MacroAssembler _masm(&cbuf);
 3860 #ifdef ASSERT
 3861     if (Compile::current()->max_vector_size() > 0) {
 3862       __ verify_ptrue();
 3863     }
 3864 #endif
 3865     __ ret(lr);
 3866   %}
 3867 
 3868   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3869     C2_MacroAssembler _masm(&cbuf);
 3870     Register target_reg = as_Register($jump_target$$reg);
 3871     __ br(target_reg);
 3872   %}
 3873 
 3874   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3875     C2_MacroAssembler _masm(&cbuf);
 3876     Register target_reg = as_Register($jump_target$$reg);
 3877     // exception oop should be in r0
 3878     // ret addr has been popped into lr
 3879     // callee expects it in r3
 3880     __ mov(r3, lr);
 3881     __ br(target_reg);
 3882   %}
 3883 
 3884   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3885     C2_MacroAssembler _masm(&cbuf);
 3886     Register oop = as_Register($object$$reg);
 3887     Register box = as_Register($box$$reg);
 3888     Register disp_hdr = as_Register($tmp$$reg);
 3889     Register tmp = as_Register($tmp2$$reg);
 3890     Label cont;
 3891     Label object_has_monitor;
 3892     Label cas_failed;
 3893 
 3894     assert_different_registers(oop, box, tmp, disp_hdr);
 3895 
 3896     // Load markWord from object into displaced_header.
 3897     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3898 
 3899     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3900       __ load_klass(tmp, oop);
 3901       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3902       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3903       __ br(Assembler::NE, cont);
 3904     }
 3905 
 3906     // Check for existing monitor
 3907     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3908 
 3909     // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3910     __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3911 
 3912     // Initialize the box. (Must happen before we update the object mark!)
 3913     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3914 
 3915     // Compare object markWord with an unlocked value (tmp) and if
 3916     // equal exchange the stack address of our box with object markWord.
 3917     // On failure disp_hdr contains the possibly locked markWord.
 3918     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3919                /*release*/ true, /*weak*/ false, disp_hdr);
 3920     __ br(Assembler::EQ, cont);
 3921 
 3922     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3923 
 3924     // If the compare-and-exchange succeeded, then we found an unlocked
 3925     // object, will have now locked it will continue at label cont
 3926 
 3927     __ bind(cas_failed);
 3928     // We did not see an unlocked object so try the fast recursive case.
 3929 
 3930     // Check if the owner is self by comparing the value in the
 3931     // markWord of object (disp_hdr) with the stack pointer.
 3932     __ mov(rscratch1, sp);
 3933     __ sub(disp_hdr, disp_hdr, rscratch1);
 3934     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3935     // If condition is true we are cont and hence we can store 0 as the
 3936     // displaced header in the box, which indicates that it is a recursive lock.
 3937     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3938     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3939 
 3940     __ b(cont);
 3941 
 3942     // Handle existing monitor.
 3943     __ bind(object_has_monitor);
 3944 
 3945     // The object's monitor m is unlocked iff m->owner == NULL,
 3946     // otherwise m->owner may contain a thread or a stack address.
 3947     //
 3948     // Try to CAS m->owner from NULL to current thread.
 3949     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3950     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3951                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3952 
 3953     // Store a non-null value into the box to avoid looking like a re-entrant
 3954     // lock. The fast-path monitor unlock code checks for
 3955     // markWord::monitor_value so use markWord::unused_mark which has the
 3956     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3957     __ mov(tmp, (address)markWord::unused_mark().value());
 3958     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3959 
 3960     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3961 
 3962     __ cmp(rscratch1, rthread);
 3963     __ br(Assembler::NE, cont); // Check for recursive locking
 3964 
 3965     // Recursive lock case
 3966     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3967     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3968 
 3969     __ bind(cont);
 3970     // flag == EQ indicates success
 3971     // flag == NE indicates failure
 3972   %}
 3973 
 3974   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3975     C2_MacroAssembler _masm(&cbuf);
 3976     Register oop = as_Register($object$$reg);
 3977     Register box = as_Register($box$$reg);
 3978     Register disp_hdr = as_Register($tmp$$reg);
 3979     Register tmp = as_Register($tmp2$$reg);
 3980     Label cont;
 3981     Label object_has_monitor;
 3982 
 3983     assert_different_registers(oop, box, tmp, disp_hdr);
 3984 
 3985     // Find the lock address and load the displaced header from the stack.
 3986     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3987 
 3988     // If the displaced header is 0, we have a recursive unlock.
 3989     __ cmp(disp_hdr, zr);
 3990     __ br(Assembler::EQ, cont);
 3991 
 3992     // Handle existing monitor.
 3993     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3994     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3995 
 3996     // Check if it is still a light weight lock, this is is true if we
 3997     // see the stack address of the basicLock in the markWord of the
 3998     // object.
 3999 
 4000     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 4001                /*release*/ true, /*weak*/ false, tmp);
 4002     __ b(cont);
 4003 
 4004     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 4005 
 4006     // Handle existing monitor.
 4007     __ bind(object_has_monitor);
 4008     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 4009     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 4010     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 4011     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4012 
 4013     Label notRecursive;
 4014     __ cmp(rscratch1, rthread);
 4015     __ br(Assembler::NE, cont);
 4016 
 4017     __ cbz(disp_hdr, notRecursive);
 4018 
 4019     // Recursive lock
 4020     __ sub(disp_hdr, disp_hdr, 1u);
 4021     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4022     // flag == EQ was set in the ownership check above
 4023     __ b(cont);
 4024 
 4025     __ bind(notRecursive);
 4026     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 4027     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 4028     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 4029     __ cmp(rscratch1, zr); // Sets flags for result
 4030     __ cbnz(rscratch1, cont);
 4031     // need a release store here
 4032     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 4033     __ stlr(zr, tmp); // set unowned
 4034 
 4035     __ bind(cont);
 4036     // flag == EQ indicates success
 4037     // flag == NE indicates failure
 4038   %}
 4039 
 4040 %}
 4041 
 4042 //----------FRAME--------------------------------------------------------------
 4043 // Definition of frame structure and management information.
 4044 //
 4045 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4046 //                             |   (to get allocators register number
 4047 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4048 //  r   CALLER     |        |
 4049 //  o     |        +--------+      pad to even-align allocators stack-slot
 4050 //  w     V        |  pad0  |        numbers; owned by CALLER
 4051 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4052 //  h     ^        |   in   |  5
 4053 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4054 //  |     |        |        |  3
 4055 //  |     |        +--------+
 4056 //  V     |        | old out|      Empty on Intel, window on Sparc
 4057 //        |    old |preserve|      Must be even aligned.
 4058 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4059 //        |        |   in   |  3   area for Intel ret address
 4060 //     Owned by    |preserve|      Empty on Sparc.
 4061 //       SELF      +--------+
 4062 //        |        |  pad2  |  2   pad to align old SP
 4063 //        |        +--------+  1
 4064 //        |        | locks  |  0
 4065 //        |        +--------+----> OptoReg::stack0(), even aligned
 4066 //        |        |  pad1  | 11   pad to align new SP
 4067 //        |        +--------+
 4068 //        |        |        | 10
 4069 //        |        | spills |  9   spills
 4070 //        V        |        |  8   (pad0 slot for callee)
 4071 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4072 //        ^        |  out   |  7
 4073 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4074 //     Owned by    +--------+
 4075 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4076 //        |    new |preserve|      Must be even-aligned.
 4077 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4078 //        |        |        |
 4079 //
 4080 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4081 //         known from SELF's arguments and the Java calling convention.
 4082 //         Region 6-7 is determined per call site.
 4083 // Note 2: If the calling convention leaves holes in the incoming argument
 4084 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4085 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4086 //         incoming area, as the Java calling convention is completely under
 4087 //         the control of the AD file.  Doubles can be sorted and packed to
 4088 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4089 //         varargs C calling conventions.
 4090 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4091 //         even aligned with pad0 as needed.
 4092 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4093 //           (the latter is true on Intel but is it false on AArch64?)
 4094 //         region 6-11 is even aligned; it may be padded out more so that
 4095 //         the region from SP to FP meets the minimum stack alignment.
 4096 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4097 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4098 //         SP meets the minimum alignment.
 4099 
 4100 frame %{
 4101   // These three registers define part of the calling convention
 4102   // between compiled code and the interpreter.
 4103 
 4104   // Inline Cache Register or Method for I2C.
 4105   inline_cache_reg(R12);
 4106 
 4107   // Number of stack slots consumed by locking an object
 4108   sync_stack_slots(2);
 4109 
 4110   // Compiled code's Frame Pointer
 4111   frame_pointer(R31);
 4112 
 4113   // Interpreter stores its frame pointer in a register which is
 4114   // stored to the stack by I2CAdaptors.
 4115   // I2CAdaptors convert from interpreted java to compiled java.
 4116   interpreter_frame_pointer(R29);
 4117 
 4118   // Stack alignment requirement
 4119   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4120 
 4121   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4122   // for calls to C.  Supports the var-args backing area for register parms.
 4123   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4124 
 4125   // The after-PROLOG location of the return address.  Location of
 4126   // return address specifies a type (REG or STACK) and a number
 4127   // representing the register number (i.e. - use a register name) or
 4128   // stack slot.
 4129   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4130   // Otherwise, it is above the locks and verification slot and alignment word
 4131   // TODO this may well be correct but need to check why that - 2 is there
 4132   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4133   // which folds in the space used for monitors
 4134   return_addr(STACK - 2 +
 4135               align_up((Compile::current()->in_preserve_stack_slots() +
 4136                         Compile::current()->fixed_slots()),
 4137                        stack_alignment_in_slots()));
 4138 
 4139   // Location of compiled Java return values.  Same as C for now.
 4140   return_value
 4141   %{
 4142     // TODO do we allow ideal_reg == Op_RegN???
 4143     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4144            "only return normal values");
 4145 
 4146     static const int lo[Op_RegL + 1] = { // enum name
 4147       0,                                 // Op_Node
 4148       0,                                 // Op_Set
 4149       R0_num,                            // Op_RegN
 4150       R0_num,                            // Op_RegI
 4151       R0_num,                            // Op_RegP
 4152       V0_num,                            // Op_RegF
 4153       V0_num,                            // Op_RegD
 4154       R0_num                             // Op_RegL
 4155     };
 4156 
 4157     static const int hi[Op_RegL + 1] = { // enum name
 4158       0,                                 // Op_Node
 4159       0,                                 // Op_Set
 4160       OptoReg::Bad,                      // Op_RegN
 4161       OptoReg::Bad,                      // Op_RegI
 4162       R0_H_num,                          // Op_RegP
 4163       OptoReg::Bad,                      // Op_RegF
 4164       V0_H_num,                          // Op_RegD
 4165       R0_H_num                           // Op_RegL
 4166     };
 4167 
 4168     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4169   %}
 4170 %}
 4171 
 4172 //----------ATTRIBUTES---------------------------------------------------------
 4173 //----------Operand Attributes-------------------------------------------------
 4174 op_attrib op_cost(1);        // Required cost attribute
 4175 
 4176 //----------Instruction Attributes---------------------------------------------
 4177 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4178 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4179 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4180                                 // a non-matching short branch variant
 4181                                 // of some long branch?
 4182 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4183                                 // be a power of 2) specifies the
 4184                                 // alignment that some part of the
 4185                                 // instruction (not necessarily the
 4186                                 // start) requires.  If > 1, a
 4187                                 // compute_padding() function must be
 4188                                 // provided for the instruction
 4189 
 4190 //----------OPERANDS-----------------------------------------------------------
 4191 // Operand definitions must precede instruction definitions for correct parsing
 4192 // in the ADLC because operands constitute user defined types which are used in
 4193 // instruction definitions.
 4194 
 4195 //----------Simple Operands----------------------------------------------------
 4196 
 4197 // Integer operands 32 bit
 4198 // 32 bit immediate
 4199 operand immI()
 4200 %{
 4201   match(ConI);
 4202 
 4203   op_cost(0);
 4204   format %{ %}
 4205   interface(CONST_INTER);
 4206 %}
 4207 
 4208 // 32 bit zero
 4209 operand immI0()
 4210 %{
 4211   predicate(n->get_int() == 0);
 4212   match(ConI);
 4213 
 4214   op_cost(0);
 4215   format %{ %}
 4216   interface(CONST_INTER);
 4217 %}
 4218 
 4219 // 32 bit unit increment
 4220 operand immI_1()
 4221 %{
 4222   predicate(n->get_int() == 1);
 4223   match(ConI);
 4224 
 4225   op_cost(0);
 4226   format %{ %}
 4227   interface(CONST_INTER);
 4228 %}
 4229 
 4230 // 32 bit unit decrement
 4231 operand immI_M1()
 4232 %{
 4233   predicate(n->get_int() == -1);
 4234   match(ConI);
 4235 
 4236   op_cost(0);
 4237   format %{ %}
 4238   interface(CONST_INTER);
 4239 %}
 4240 
 4241 // Shift values for add/sub extension shift
 4242 operand immIExt()
 4243 %{
 4244   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4245   match(ConI);
 4246 
 4247   op_cost(0);
 4248   format %{ %}
 4249   interface(CONST_INTER);
 4250 %}
 4251 
 4252 operand immI_gt_1()
 4253 %{
 4254   predicate(n->get_int() > 1);
 4255   match(ConI);
 4256 
 4257   op_cost(0);
 4258   format %{ %}
 4259   interface(CONST_INTER);
 4260 %}
 4261 
 4262 operand immI_le_4()
 4263 %{
 4264   predicate(n->get_int() <= 4);
 4265   match(ConI);
 4266 
 4267   op_cost(0);
 4268   format %{ %}
 4269   interface(CONST_INTER);
 4270 %}
 4271 
 4272 operand immI_31()
 4273 %{
 4274   predicate(n->get_int() == 31);
 4275   match(ConI);
 4276 
 4277   op_cost(0);
 4278   format %{ %}
 4279   interface(CONST_INTER);
 4280 %}
 4281 
 4282 operand immI_2()
 4283 %{
 4284   predicate(n->get_int() == 2);
 4285   match(ConI);
 4286 
 4287   op_cost(0);
 4288   format %{ %}
 4289   interface(CONST_INTER);
 4290 %}
 4291 
 4292 operand immI_4()
 4293 %{
 4294   predicate(n->get_int() == 4);
 4295   match(ConI);
 4296 
 4297   op_cost(0);
 4298   format %{ %}
 4299   interface(CONST_INTER);
 4300 %}
 4301 
 4302 operand immI_8()
 4303 %{
 4304   predicate(n->get_int() == 8);
 4305   match(ConI);
 4306 
 4307   op_cost(0);
 4308   format %{ %}
 4309   interface(CONST_INTER);
 4310 %}
 4311 
 4312 operand immI_16()
 4313 %{
 4314   predicate(n->get_int() == 16);
 4315   match(ConI);
 4316 
 4317   op_cost(0);
 4318   format %{ %}
 4319   interface(CONST_INTER);
 4320 %}
 4321 
 4322 operand immI_24()
 4323 %{
 4324   predicate(n->get_int() == 24);
 4325   match(ConI);
 4326 
 4327   op_cost(0);
 4328   format %{ %}
 4329   interface(CONST_INTER);
 4330 %}
 4331 
 4332 operand immI_32()
 4333 %{
 4334   predicate(n->get_int() == 32);
 4335   match(ConI);
 4336 
 4337   op_cost(0);
 4338   format %{ %}
 4339   interface(CONST_INTER);
 4340 %}
 4341 
 4342 operand immI_48()
 4343 %{
 4344   predicate(n->get_int() == 48);
 4345   match(ConI);
 4346 
 4347   op_cost(0);
 4348   format %{ %}
 4349   interface(CONST_INTER);
 4350 %}
 4351 
 4352 operand immI_56()
 4353 %{
 4354   predicate(n->get_int() == 56);
 4355   match(ConI);
 4356 
 4357   op_cost(0);
 4358   format %{ %}
 4359   interface(CONST_INTER);
 4360 %}
 4361 
 4362 operand immI_63()
 4363 %{
 4364   predicate(n->get_int() == 63);
 4365   match(ConI);
 4366 
 4367   op_cost(0);
 4368   format %{ %}
 4369   interface(CONST_INTER);
 4370 %}
 4371 
 4372 operand immI_64()
 4373 %{
 4374   predicate(n->get_int() == 64);
 4375   match(ConI);
 4376 
 4377   op_cost(0);
 4378   format %{ %}
 4379   interface(CONST_INTER);
 4380 %}
 4381 
 4382 operand immI_255()
 4383 %{
 4384   predicate(n->get_int() == 255);
 4385   match(ConI);
 4386 
 4387   op_cost(0);
 4388   format %{ %}
 4389   interface(CONST_INTER);
 4390 %}
 4391 
 4392 operand immI_65535()
 4393 %{
 4394   predicate(n->get_int() == 65535);
 4395   match(ConI);
 4396 
 4397   op_cost(0);
 4398   format %{ %}
 4399   interface(CONST_INTER);
 4400 %}
 4401 
 4402 operand immL_255()
 4403 %{
 4404   predicate(n->get_long() == 255L);
 4405   match(ConL);
 4406 
 4407   op_cost(0);
 4408   format %{ %}
 4409   interface(CONST_INTER);
 4410 %}
 4411 
 4412 operand immL_65535()
 4413 %{
 4414   predicate(n->get_long() == 65535L);
 4415   match(ConL);
 4416 
 4417   op_cost(0);
 4418   format %{ %}
 4419   interface(CONST_INTER);
 4420 %}
 4421 
 4422 operand immL_4294967295()
 4423 %{
 4424   predicate(n->get_long() == 4294967295L);
 4425   match(ConL);
 4426 
 4427   op_cost(0);
 4428   format %{ %}
 4429   interface(CONST_INTER);
 4430 %}
 4431 
 4432 operand immL_bitmask()
 4433 %{
 4434   predicate((n->get_long() != 0)
 4435             && ((n->get_long() & 0xc000000000000000l) == 0)
 4436             && is_power_of_2(n->get_long() + 1));
 4437   match(ConL);
 4438 
 4439   op_cost(0);
 4440   format %{ %}
 4441   interface(CONST_INTER);
 4442 %}
 4443 
 4444 operand immI_bitmask()
 4445 %{
 4446   predicate((n->get_int() != 0)
 4447             && ((n->get_int() & 0xc0000000) == 0)
 4448             && is_power_of_2(n->get_int() + 1));
 4449   match(ConI);
 4450 
 4451   op_cost(0);
 4452   format %{ %}
 4453   interface(CONST_INTER);
 4454 %}
 4455 
 4456 operand immL_positive_bitmaskI()
 4457 %{
 4458   predicate((n->get_long() != 0)
 4459             && ((julong)n->get_long() < 0x80000000ULL)
 4460             && is_power_of_2(n->get_long() + 1));
 4461   match(ConL);
 4462 
 4463   op_cost(0);
 4464   format %{ %}
 4465   interface(CONST_INTER);
 4466 %}
 4467 
 4468 // Scale values for scaled offset addressing modes (up to long but not quad)
 4469 operand immIScale()
 4470 %{
 4471   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4472   match(ConI);
 4473 
 4474   op_cost(0);
 4475   format %{ %}
 4476   interface(CONST_INTER);
 4477 %}
 4478 
 4479 // 26 bit signed offset -- for pc-relative branches
 4480 operand immI26()
 4481 %{
 4482   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4483   match(ConI);
 4484 
 4485   op_cost(0);
 4486   format %{ %}
 4487   interface(CONST_INTER);
 4488 %}
 4489 
 4490 // 19 bit signed offset -- for pc-relative loads
 4491 operand immI19()
 4492 %{
 4493   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4494   match(ConI);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 // 12 bit unsigned offset -- for base plus immediate loads
 4502 operand immIU12()
 4503 %{
 4504   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4505   match(ConI);
 4506 
 4507   op_cost(0);
 4508   format %{ %}
 4509   interface(CONST_INTER);
 4510 %}
 4511 
 4512 operand immLU12()
 4513 %{
 4514   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4515   match(ConL);
 4516 
 4517   op_cost(0);
 4518   format %{ %}
 4519   interface(CONST_INTER);
 4520 %}
 4521 
 4522 // Offset for scaled or unscaled immediate loads and stores
 4523 operand immIOffset()
 4524 %{
 4525   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4526   match(ConI);
 4527 
 4528   op_cost(0);
 4529   format %{ %}
 4530   interface(CONST_INTER);
 4531 %}
 4532 
 4533 operand immIOffset1()
 4534 %{
 4535   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4536   match(ConI);
 4537 
 4538   op_cost(0);
 4539   format %{ %}
 4540   interface(CONST_INTER);
 4541 %}
 4542 
 4543 operand immIOffset2()
 4544 %{
 4545   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4546   match(ConI);
 4547 
 4548   op_cost(0);
 4549   format %{ %}
 4550   interface(CONST_INTER);
 4551 %}
 4552 
 4553 operand immIOffset4()
 4554 %{
 4555   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4556   match(ConI);
 4557 
 4558   op_cost(0);
 4559   format %{ %}
 4560   interface(CONST_INTER);
 4561 %}
 4562 
 4563 operand immIOffset8()
 4564 %{
 4565   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4566   match(ConI);
 4567 
 4568   op_cost(0);
 4569   format %{ %}
 4570   interface(CONST_INTER);
 4571 %}
 4572 
 4573 operand immIOffset16()
 4574 %{
 4575   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4576   match(ConI);
 4577 
 4578   op_cost(0);
 4579   format %{ %}
 4580   interface(CONST_INTER);
 4581 %}
 4582 
 4583 operand immLoffset()
 4584 %{
 4585   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4586   match(ConL);
 4587 
 4588   op_cost(0);
 4589   format %{ %}
 4590   interface(CONST_INTER);
 4591 %}
 4592 
 4593 operand immLoffset1()
 4594 %{
 4595   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4596   match(ConL);
 4597 
 4598   op_cost(0);
 4599   format %{ %}
 4600   interface(CONST_INTER);
 4601 %}
 4602 
 4603 operand immLoffset2()
 4604 %{
 4605   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4606   match(ConL);
 4607 
 4608   op_cost(0);
 4609   format %{ %}
 4610   interface(CONST_INTER);
 4611 %}
 4612 
 4613 operand immLoffset4()
 4614 %{
 4615   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4616   match(ConL);
 4617 
 4618   op_cost(0);
 4619   format %{ %}
 4620   interface(CONST_INTER);
 4621 %}
 4622 
 4623 operand immLoffset8()
 4624 %{
 4625   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4626   match(ConL);
 4627 
 4628   op_cost(0);
 4629   format %{ %}
 4630   interface(CONST_INTER);
 4631 %}
 4632 
 4633 operand immLoffset16()
 4634 %{
 4635   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4636   match(ConL);
 4637 
 4638   op_cost(0);
 4639   format %{ %}
 4640   interface(CONST_INTER);
 4641 %}
 4642 
 4643 // 8 bit signed value.
 4644 operand immI8()
 4645 %{
 4646   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4647   match(ConI);
 4648 
 4649   op_cost(0);
 4650   format %{ %}
 4651   interface(CONST_INTER);
 4652 %}
 4653 
 4654 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4655 operand immI8_shift8()
 4656 %{
 4657   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4658             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4659   match(ConI);
 4660 
 4661   op_cost(0);
 4662   format %{ %}
 4663   interface(CONST_INTER);
 4664 %}
 4665 
 4666 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4667 operand immL8_shift8()
 4668 %{
 4669   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4670             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4671   match(ConL);
 4672 
 4673   op_cost(0);
 4674   format %{ %}
 4675   interface(CONST_INTER);
 4676 %}
 4677 
 4678 // 8 bit integer valid for vector add sub immediate
 4679 operand immBAddSubV()
 4680 %{
 4681   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4682   match(ConI);
 4683 
 4684   op_cost(0);
 4685   format %{ %}
 4686   interface(CONST_INTER);
 4687 %}
 4688 
 4689 // 32 bit integer valid for add sub immediate
 4690 operand immIAddSub()
 4691 %{
 4692   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4693   match(ConI);
 4694   op_cost(0);
 4695   format %{ %}
 4696   interface(CONST_INTER);
 4697 %}
 4698 
 4699 // 32 bit integer valid for vector add sub immediate
 4700 operand immIAddSubV()
 4701 %{
 4702   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4703   match(ConI);
 4704 
 4705   op_cost(0);
 4706   format %{ %}
 4707   interface(CONST_INTER);
 4708 %}
 4709 
 4710 // 32 bit unsigned integer valid for logical immediate
 4711 
 4712 operand immBLog()
 4713 %{
 4714   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4715   match(ConI);
 4716 
 4717   op_cost(0);
 4718   format %{ %}
 4719   interface(CONST_INTER);
 4720 %}
 4721 
 4722 operand immSLog()
 4723 %{
 4724   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4725   match(ConI);
 4726 
 4727   op_cost(0);
 4728   format %{ %}
 4729   interface(CONST_INTER);
 4730 %}
 4731 
 4732 operand immILog()
 4733 %{
 4734   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4735   match(ConI);
 4736 
 4737   op_cost(0);
 4738   format %{ %}
 4739   interface(CONST_INTER);
 4740 %}
 4741 
 4742 // Integer operands 64 bit
 4743 // 64 bit immediate
 4744 operand immL()
 4745 %{
 4746   match(ConL);
 4747 
 4748   op_cost(0);
 4749   format %{ %}
 4750   interface(CONST_INTER);
 4751 %}
 4752 
 4753 // 64 bit zero
 4754 operand immL0()
 4755 %{
 4756   predicate(n->get_long() == 0);
 4757   match(ConL);
 4758 
 4759   op_cost(0);
 4760   format %{ %}
 4761   interface(CONST_INTER);
 4762 %}
 4763 
 4764 // 64 bit unit increment
 4765 operand immL_1()
 4766 %{
 4767   predicate(n->get_long() == 1);
 4768   match(ConL);
 4769 
 4770   op_cost(0);
 4771   format %{ %}
 4772   interface(CONST_INTER);
 4773 %}
 4774 
 4775 // 64 bit unit decrement
 4776 operand immL_M1()
 4777 %{
 4778   predicate(n->get_long() == -1);
 4779   match(ConL);
 4780 
 4781   op_cost(0);
 4782   format %{ %}
 4783   interface(CONST_INTER);
 4784 %}
 4785 
 4786 // 32 bit offset of pc in thread anchor
 4787 
 4788 operand immL_pc_off()
 4789 %{
 4790   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4791                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4792   match(ConL);
 4793 
 4794   op_cost(0);
 4795   format %{ %}
 4796   interface(CONST_INTER);
 4797 %}
 4798 
 4799 // 64 bit integer valid for add sub immediate
 4800 operand immLAddSub()
 4801 %{
 4802   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4803   match(ConL);
 4804   op_cost(0);
 4805   format %{ %}
 4806   interface(CONST_INTER);
 4807 %}
 4808 
 4809 // 64 bit integer valid for addv subv immediate
 4810 operand immLAddSubV()
 4811 %{
 4812   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4813   match(ConL);
 4814 
 4815   op_cost(0);
 4816   format %{ %}
 4817   interface(CONST_INTER);
 4818 %}
 4819 
 4820 // 64 bit integer valid for logical immediate
 4821 operand immLLog()
 4822 %{
 4823   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4824   match(ConL);
 4825   op_cost(0);
 4826   format %{ %}
 4827   interface(CONST_INTER);
 4828 %}
 4829 
 4830 // Long Immediate: low 32-bit mask
 4831 operand immL_32bits()
 4832 %{
 4833   predicate(n->get_long() == 0xFFFFFFFFL);
 4834   match(ConL);
 4835   op_cost(0);
 4836   format %{ %}
 4837   interface(CONST_INTER);
 4838 %}
 4839 
 4840 // Pointer operands
 4841 // Pointer Immediate
 4842 operand immP()
 4843 %{
 4844   match(ConP);
 4845 
 4846   op_cost(0);
 4847   format %{ %}
 4848   interface(CONST_INTER);
 4849 %}
 4850 
 4851 // NULL Pointer Immediate
 4852 operand immP0()
 4853 %{
 4854   predicate(n->get_ptr() == 0);
 4855   match(ConP);
 4856 
 4857   op_cost(0);
 4858   format %{ %}
 4859   interface(CONST_INTER);
 4860 %}
 4861 
 4862 // Pointer Immediate One
 4863 // this is used in object initialization (initial object header)
 4864 operand immP_1()
 4865 %{
 4866   predicate(n->get_ptr() == 1);
 4867   match(ConP);
 4868 
 4869   op_cost(0);
 4870   format %{ %}
 4871   interface(CONST_INTER);
 4872 %}
 4873 
 4874 // Card Table Byte Map Base
 4875 operand immByteMapBase()
 4876 %{
 4877   // Get base of card map
 4878   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4879             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4880   match(ConP);
 4881 
 4882   op_cost(0);
 4883   format %{ %}
 4884   interface(CONST_INTER);
 4885 %}
 4886 
 4887 // Pointer Immediate Minus One
 4888 // this is used when we want to write the current PC to the thread anchor
 4889 operand immP_M1()
 4890 %{
 4891   predicate(n->get_ptr() == -1);
 4892   match(ConP);
 4893 
 4894   op_cost(0);
 4895   format %{ %}
 4896   interface(CONST_INTER);
 4897 %}
 4898 
 4899 // Pointer Immediate Minus Two
 4900 // this is used when we want to write the current PC to the thread anchor
 4901 operand immP_M2()
 4902 %{
 4903   predicate(n->get_ptr() == -2);
 4904   match(ConP);
 4905 
 4906   op_cost(0);
 4907   format %{ %}
 4908   interface(CONST_INTER);
 4909 %}
 4910 
 4911 // Float and Double operands
 4912 // Double Immediate
 4913 operand immD()
 4914 %{
 4915   match(ConD);
 4916   op_cost(0);
 4917   format %{ %}
 4918   interface(CONST_INTER);
 4919 %}
 4920 
 4921 // Double Immediate: +0.0d
 4922 operand immD0()
 4923 %{
 4924   predicate(jlong_cast(n->getd()) == 0);
 4925   match(ConD);
 4926 
 4927   op_cost(0);
 4928   format %{ %}
 4929   interface(CONST_INTER);
 4930 %}
 4931 
 4932 // constant 'double +0.0'.
 4933 operand immDPacked()
 4934 %{
 4935   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4936   match(ConD);
 4937   op_cost(0);
 4938   format %{ %}
 4939   interface(CONST_INTER);
 4940 %}
 4941 
 4942 // Float Immediate
 4943 operand immF()
 4944 %{
 4945   match(ConF);
 4946   op_cost(0);
 4947   format %{ %}
 4948   interface(CONST_INTER);
 4949 %}
 4950 
 4951 // Float Immediate: +0.0f.
 4952 operand immF0()
 4953 %{
 4954   predicate(jint_cast(n->getf()) == 0);
 4955   match(ConF);
 4956 
 4957   op_cost(0);
 4958   format %{ %}
 4959   interface(CONST_INTER);
 4960 %}
 4961 
 4962 //
 4963 operand immFPacked()
 4964 %{
 4965   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4966   match(ConF);
 4967   op_cost(0);
 4968   format %{ %}
 4969   interface(CONST_INTER);
 4970 %}
 4971 
 4972 // Narrow pointer operands
 4973 // Narrow Pointer Immediate
 4974 operand immN()
 4975 %{
 4976   match(ConN);
 4977 
 4978   op_cost(0);
 4979   format %{ %}
 4980   interface(CONST_INTER);
 4981 %}
 4982 
 4983 // Narrow NULL Pointer Immediate
 4984 operand immN0()
 4985 %{
 4986   predicate(n->get_narrowcon() == 0);
 4987   match(ConN);
 4988 
 4989   op_cost(0);
 4990   format %{ %}
 4991   interface(CONST_INTER);
 4992 %}
 4993 
 4994 operand immNKlass()
 4995 %{
 4996   match(ConNKlass);
 4997 
 4998   op_cost(0);
 4999   format %{ %}
 5000   interface(CONST_INTER);
 5001 %}
 5002 
 5003 // Integer 32 bit Register Operands
 5004 // Integer 32 bitRegister (excludes SP)
 5005 operand iRegI()
 5006 %{
 5007   constraint(ALLOC_IN_RC(any_reg32));
 5008   match(RegI);
 5009   match(iRegINoSp);
 5010   op_cost(0);
 5011   format %{ %}
 5012   interface(REG_INTER);
 5013 %}
 5014 
 5015 // Integer 32 bit Register not Special
 5016 operand iRegINoSp()
 5017 %{
 5018   constraint(ALLOC_IN_RC(no_special_reg32));
 5019   match(RegI);
 5020   op_cost(0);
 5021   format %{ %}
 5022   interface(REG_INTER);
 5023 %}
 5024 
 5025 // Integer 64 bit Register Operands
 5026 // Integer 64 bit Register (includes SP)
 5027 operand iRegL()
 5028 %{
 5029   constraint(ALLOC_IN_RC(any_reg));
 5030   match(RegL);
 5031   match(iRegLNoSp);
 5032   op_cost(0);
 5033   format %{ %}
 5034   interface(REG_INTER);
 5035 %}
 5036 
 5037 // Integer 64 bit Register not Special
 5038 operand iRegLNoSp()
 5039 %{
 5040   constraint(ALLOC_IN_RC(no_special_reg));
 5041   match(RegL);
 5042   match(iRegL_R0);
 5043   format %{ %}
 5044   interface(REG_INTER);
 5045 %}
 5046 
 5047 // Pointer Register Operands
 5048 // Pointer Register
 5049 operand iRegP()
 5050 %{
 5051   constraint(ALLOC_IN_RC(ptr_reg));
 5052   match(RegP);
 5053   match(iRegPNoSp);
 5054   match(iRegP_R0);
 5055   //match(iRegP_R2);
 5056   //match(iRegP_R4);
 5057   //match(iRegP_R5);
 5058   match(thread_RegP);
 5059   op_cost(0);
 5060   format %{ %}
 5061   interface(REG_INTER);
 5062 %}
 5063 
 5064 // Pointer 64 bit Register not Special
 5065 operand iRegPNoSp()
 5066 %{
 5067   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 5068   match(RegP);
 5069   // match(iRegP);
 5070   // match(iRegP_R0);
 5071   // match(iRegP_R2);
 5072   // match(iRegP_R4);
 5073   // match(iRegP_R5);
 5074   // match(thread_RegP);
 5075   op_cost(0);
 5076   format %{ %}
 5077   interface(REG_INTER);
 5078 %}
 5079 
 5080 // Pointer 64 bit Register R0 only
 5081 operand iRegP_R0()
 5082 %{
 5083   constraint(ALLOC_IN_RC(r0_reg));
 5084   match(RegP);
 5085   // match(iRegP);
 5086   match(iRegPNoSp);
 5087   op_cost(0);
 5088   format %{ %}
 5089   interface(REG_INTER);
 5090 %}
 5091 
 5092 // Pointer 64 bit Register R1 only
 5093 operand iRegP_R1()
 5094 %{
 5095   constraint(ALLOC_IN_RC(r1_reg));
 5096   match(RegP);
 5097   // match(iRegP);
 5098   match(iRegPNoSp);
 5099   op_cost(0);
 5100   format %{ %}
 5101   interface(REG_INTER);
 5102 %}
 5103 
 5104 // Pointer 64 bit Register R2 only
 5105 operand iRegP_R2()
 5106 %{
 5107   constraint(ALLOC_IN_RC(r2_reg));
 5108   match(RegP);
 5109   // match(iRegP);
 5110   match(iRegPNoSp);
 5111   op_cost(0);
 5112   format %{ %}
 5113   interface(REG_INTER);
 5114 %}
 5115 
 5116 // Pointer 64 bit Register R3 only
 5117 operand iRegP_R3()
 5118 %{
 5119   constraint(ALLOC_IN_RC(r3_reg));
 5120   match(RegP);
 5121   // match(iRegP);
 5122   match(iRegPNoSp);
 5123   op_cost(0);
 5124   format %{ %}
 5125   interface(REG_INTER);
 5126 %}
 5127 
 5128 // Pointer 64 bit Register R4 only
 5129 operand iRegP_R4()
 5130 %{
 5131   constraint(ALLOC_IN_RC(r4_reg));
 5132   match(RegP);
 5133   // match(iRegP);
 5134   match(iRegPNoSp);
 5135   op_cost(0);
 5136   format %{ %}
 5137   interface(REG_INTER);
 5138 %}
 5139 
 5140 // Pointer 64 bit Register R5 only
 5141 operand iRegP_R5()
 5142 %{
 5143   constraint(ALLOC_IN_RC(r5_reg));
 5144   match(RegP);
 5145   // match(iRegP);
 5146   match(iRegPNoSp);
 5147   op_cost(0);
 5148   format %{ %}
 5149   interface(REG_INTER);
 5150 %}
 5151 
 5152 // Pointer 64 bit Register R10 only
 5153 operand iRegP_R10()
 5154 %{
 5155   constraint(ALLOC_IN_RC(r10_reg));
 5156   match(RegP);
 5157   // match(iRegP);
 5158   match(iRegPNoSp);
 5159   op_cost(0);
 5160   format %{ %}
 5161   interface(REG_INTER);
 5162 %}
 5163 
 5164 // Long 64 bit Register R0 only
 5165 operand iRegL_R0()
 5166 %{
 5167   constraint(ALLOC_IN_RC(r0_reg));
 5168   match(RegL);
 5169   match(iRegLNoSp);
 5170   op_cost(0);
 5171   format %{ %}
 5172   interface(REG_INTER);
 5173 %}
 5174 
 5175 // Long 64 bit Register R2 only
 5176 operand iRegL_R2()
 5177 %{
 5178   constraint(ALLOC_IN_RC(r2_reg));
 5179   match(RegL);
 5180   match(iRegLNoSp);
 5181   op_cost(0);
 5182   format %{ %}
 5183   interface(REG_INTER);
 5184 %}
 5185 
 5186 // Long 64 bit Register R3 only
 5187 operand iRegL_R3()
 5188 %{
 5189   constraint(ALLOC_IN_RC(r3_reg));
 5190   match(RegL);
 5191   match(iRegLNoSp);
 5192   op_cost(0);
 5193   format %{ %}
 5194   interface(REG_INTER);
 5195 %}
 5196 
 5197 // Long 64 bit Register R11 only
 5198 operand iRegL_R11()
 5199 %{
 5200   constraint(ALLOC_IN_RC(r11_reg));
 5201   match(RegL);
 5202   match(iRegLNoSp);
 5203   op_cost(0);
 5204   format %{ %}
 5205   interface(REG_INTER);
 5206 %}
 5207 
 5208 // Pointer 64 bit Register FP only
 5209 operand iRegP_FP()
 5210 %{
 5211   constraint(ALLOC_IN_RC(fp_reg));
 5212   match(RegP);
 5213   // match(iRegP);
 5214   op_cost(0);
 5215   format %{ %}
 5216   interface(REG_INTER);
 5217 %}
 5218 
 5219 // Register R0 only
 5220 operand iRegI_R0()
 5221 %{
 5222   constraint(ALLOC_IN_RC(int_r0_reg));
 5223   match(RegI);
 5224   match(iRegINoSp);
 5225   op_cost(0);
 5226   format %{ %}
 5227   interface(REG_INTER);
 5228 %}
 5229 
 5230 // Register R2 only
 5231 operand iRegI_R2()
 5232 %{
 5233   constraint(ALLOC_IN_RC(int_r2_reg));
 5234   match(RegI);
 5235   match(iRegINoSp);
 5236   op_cost(0);
 5237   format %{ %}
 5238   interface(REG_INTER);
 5239 %}
 5240 
 5241 // Register R3 only
 5242 operand iRegI_R3()
 5243 %{
 5244   constraint(ALLOC_IN_RC(int_r3_reg));
 5245   match(RegI);
 5246   match(iRegINoSp);
 5247   op_cost(0);
 5248   format %{ %}
 5249   interface(REG_INTER);
 5250 %}
 5251 
 5252 
 5253 // Register R4 only
 5254 operand iRegI_R4()
 5255 %{
 5256   constraint(ALLOC_IN_RC(int_r4_reg));
 5257   match(RegI);
 5258   match(iRegINoSp);
 5259   op_cost(0);
 5260   format %{ %}
 5261   interface(REG_INTER);
 5262 %}
 5263 
 5264 
 5265 // Pointer Register Operands
 5266 // Narrow Pointer Register
 5267 operand iRegN()
 5268 %{
 5269   constraint(ALLOC_IN_RC(any_reg32));
 5270   match(RegN);
 5271   match(iRegNNoSp);
 5272   op_cost(0);
 5273   format %{ %}
 5274   interface(REG_INTER);
 5275 %}
 5276 
 5277 operand iRegN_R0()
 5278 %{
 5279   constraint(ALLOC_IN_RC(r0_reg));
 5280   match(iRegN);
 5281   op_cost(0);
 5282   format %{ %}
 5283   interface(REG_INTER);
 5284 %}
 5285 
 5286 operand iRegN_R2()
 5287 %{
 5288   constraint(ALLOC_IN_RC(r2_reg));
 5289   match(iRegN);
 5290   op_cost(0);
 5291   format %{ %}
 5292   interface(REG_INTER);
 5293 %}
 5294 
 5295 operand iRegN_R3()
 5296 %{
 5297   constraint(ALLOC_IN_RC(r3_reg));
 5298   match(iRegN);
 5299   op_cost(0);
 5300   format %{ %}
 5301   interface(REG_INTER);
 5302 %}
 5303 
 5304 // Integer 64 bit Register not Special
 5305 operand iRegNNoSp()
 5306 %{
 5307   constraint(ALLOC_IN_RC(no_special_reg32));
 5308   match(RegN);
 5309   op_cost(0);
 5310   format %{ %}
 5311   interface(REG_INTER);
 5312 %}
 5313 
 5314 // heap base register -- used for encoding immN0
 5315 
 5316 operand iRegIHeapbase()
 5317 %{
 5318   constraint(ALLOC_IN_RC(heapbase_reg));
 5319   match(RegI);
 5320   op_cost(0);
 5321   format %{ %}
 5322   interface(REG_INTER);
 5323 %}
 5324 
 5325 // Float Register
 5326 // Float register operands
 5327 operand vRegF()
 5328 %{
 5329   constraint(ALLOC_IN_RC(float_reg));
 5330   match(RegF);
 5331 
 5332   op_cost(0);
 5333   format %{ %}
 5334   interface(REG_INTER);
 5335 %}
 5336 
 5337 // Double Register
 5338 // Double register operands
 5339 operand vRegD()
 5340 %{
 5341   constraint(ALLOC_IN_RC(double_reg));
 5342   match(RegD);
 5343 
 5344   op_cost(0);
 5345   format %{ %}
 5346   interface(REG_INTER);
 5347 %}
 5348 
 5349 // Generic vector class. This will be used for
 5350 // all vector operands, including NEON and SVE,
 5351 // but currently only used for SVE VecA.
 5352 operand vReg()
 5353 %{
 5354   constraint(ALLOC_IN_RC(vectora_reg));
 5355   match(VecA);
 5356   op_cost(0);
 5357   format %{ %}
 5358   interface(REG_INTER);
 5359 %}
 5360 
 5361 operand vecD()
 5362 %{
 5363   constraint(ALLOC_IN_RC(vectord_reg));
 5364   match(VecD);
 5365 
 5366   op_cost(0);
 5367   format %{ %}
 5368   interface(REG_INTER);
 5369 %}
 5370 
 5371 operand vecX()
 5372 %{
 5373   constraint(ALLOC_IN_RC(vectorx_reg));
 5374   match(VecX);
 5375 
 5376   op_cost(0);
 5377   format %{ %}
 5378   interface(REG_INTER);
 5379 %}
 5380 
 5381 operand vRegD_V0()
 5382 %{
 5383   constraint(ALLOC_IN_RC(v0_reg));
 5384   match(RegD);
 5385   op_cost(0);
 5386   format %{ %}
 5387   interface(REG_INTER);
 5388 %}
 5389 
 5390 operand vRegD_V1()
 5391 %{
 5392   constraint(ALLOC_IN_RC(v1_reg));
 5393   match(RegD);
 5394   op_cost(0);
 5395   format %{ %}
 5396   interface(REG_INTER);
 5397 %}
 5398 
 5399 operand vRegD_V2()
 5400 %{
 5401   constraint(ALLOC_IN_RC(v2_reg));
 5402   match(RegD);
 5403   op_cost(0);
 5404   format %{ %}
 5405   interface(REG_INTER);
 5406 %}
 5407 
 5408 operand vRegD_V3()
 5409 %{
 5410   constraint(ALLOC_IN_RC(v3_reg));
 5411   match(RegD);
 5412   op_cost(0);
 5413   format %{ %}
 5414   interface(REG_INTER);
 5415 %}
 5416 
 5417 operand vRegD_V4()
 5418 %{
 5419   constraint(ALLOC_IN_RC(v4_reg));
 5420   match(RegD);
 5421   op_cost(0);
 5422   format %{ %}
 5423   interface(REG_INTER);
 5424 %}
 5425 
 5426 operand vRegD_V5()
 5427 %{
 5428   constraint(ALLOC_IN_RC(v5_reg));
 5429   match(RegD);
 5430   op_cost(0);
 5431   format %{ %}
 5432   interface(REG_INTER);
 5433 %}
 5434 
 5435 operand vRegD_V6()
 5436 %{
 5437   constraint(ALLOC_IN_RC(v6_reg));
 5438   match(RegD);
 5439   op_cost(0);
 5440   format %{ %}
 5441   interface(REG_INTER);
 5442 %}
 5443 
 5444 operand vRegD_V7()
 5445 %{
 5446   constraint(ALLOC_IN_RC(v7_reg));
 5447   match(RegD);
 5448   op_cost(0);
 5449   format %{ %}
 5450   interface(REG_INTER);
 5451 %}
 5452 
 5453 operand vRegD_V8()
 5454 %{
 5455   constraint(ALLOC_IN_RC(v8_reg));
 5456   match(RegD);
 5457   op_cost(0);
 5458   format %{ %}
 5459   interface(REG_INTER);
 5460 %}
 5461 
 5462 operand vRegD_V9()
 5463 %{
 5464   constraint(ALLOC_IN_RC(v9_reg));
 5465   match(RegD);
 5466   op_cost(0);
 5467   format %{ %}
 5468   interface(REG_INTER);
 5469 %}
 5470 
 5471 operand vRegD_V10()
 5472 %{
 5473   constraint(ALLOC_IN_RC(v10_reg));
 5474   match(RegD);
 5475   op_cost(0);
 5476   format %{ %}
 5477   interface(REG_INTER);
 5478 %}
 5479 
 5480 operand vRegD_V11()
 5481 %{
 5482   constraint(ALLOC_IN_RC(v11_reg));
 5483   match(RegD);
 5484   op_cost(0);
 5485   format %{ %}
 5486   interface(REG_INTER);
 5487 %}
 5488 
 5489 operand vRegD_V12()
 5490 %{
 5491   constraint(ALLOC_IN_RC(v12_reg));
 5492   match(RegD);
 5493   op_cost(0);
 5494   format %{ %}
 5495   interface(REG_INTER);
 5496 %}
 5497 
 5498 operand vRegD_V13()
 5499 %{
 5500   constraint(ALLOC_IN_RC(v13_reg));
 5501   match(RegD);
 5502   op_cost(0);
 5503   format %{ %}
 5504   interface(REG_INTER);
 5505 %}
 5506 
 5507 operand vRegD_V14()
 5508 %{
 5509   constraint(ALLOC_IN_RC(v14_reg));
 5510   match(RegD);
 5511   op_cost(0);
 5512   format %{ %}
 5513   interface(REG_INTER);
 5514 %}
 5515 
 5516 operand vRegD_V15()
 5517 %{
 5518   constraint(ALLOC_IN_RC(v15_reg));
 5519   match(RegD);
 5520   op_cost(0);
 5521   format %{ %}
 5522   interface(REG_INTER);
 5523 %}
 5524 
 5525 operand vRegD_V16()
 5526 %{
 5527   constraint(ALLOC_IN_RC(v16_reg));
 5528   match(RegD);
 5529   op_cost(0);
 5530   format %{ %}
 5531   interface(REG_INTER);
 5532 %}
 5533 
 5534 operand vRegD_V17()
 5535 %{
 5536   constraint(ALLOC_IN_RC(v17_reg));
 5537   match(RegD);
 5538   op_cost(0);
 5539   format %{ %}
 5540   interface(REG_INTER);
 5541 %}
 5542 
 5543 operand vRegD_V18()
 5544 %{
 5545   constraint(ALLOC_IN_RC(v18_reg));
 5546   match(RegD);
 5547   op_cost(0);
 5548   format %{ %}
 5549   interface(REG_INTER);
 5550 %}
 5551 
 5552 operand vRegD_V19()
 5553 %{
 5554   constraint(ALLOC_IN_RC(v19_reg));
 5555   match(RegD);
 5556   op_cost(0);
 5557   format %{ %}
 5558   interface(REG_INTER);
 5559 %}
 5560 
 5561 operand vRegD_V20()
 5562 %{
 5563   constraint(ALLOC_IN_RC(v20_reg));
 5564   match(RegD);
 5565   op_cost(0);
 5566   format %{ %}
 5567   interface(REG_INTER);
 5568 %}
 5569 
 5570 operand vRegD_V21()
 5571 %{
 5572   constraint(ALLOC_IN_RC(v21_reg));
 5573   match(RegD);
 5574   op_cost(0);
 5575   format %{ %}
 5576   interface(REG_INTER);
 5577 %}
 5578 
 5579 operand vRegD_V22()
 5580 %{
 5581   constraint(ALLOC_IN_RC(v22_reg));
 5582   match(RegD);
 5583   op_cost(0);
 5584   format %{ %}
 5585   interface(REG_INTER);
 5586 %}
 5587 
 5588 operand vRegD_V23()
 5589 %{
 5590   constraint(ALLOC_IN_RC(v23_reg));
 5591   match(RegD);
 5592   op_cost(0);
 5593   format %{ %}
 5594   interface(REG_INTER);
 5595 %}
 5596 
 5597 operand vRegD_V24()
 5598 %{
 5599   constraint(ALLOC_IN_RC(v24_reg));
 5600   match(RegD);
 5601   op_cost(0);
 5602   format %{ %}
 5603   interface(REG_INTER);
 5604 %}
 5605 
 5606 operand vRegD_V25()
 5607 %{
 5608   constraint(ALLOC_IN_RC(v25_reg));
 5609   match(RegD);
 5610   op_cost(0);
 5611   format %{ %}
 5612   interface(REG_INTER);
 5613 %}
 5614 
 5615 operand vRegD_V26()
 5616 %{
 5617   constraint(ALLOC_IN_RC(v26_reg));
 5618   match(RegD);
 5619   op_cost(0);
 5620   format %{ %}
 5621   interface(REG_INTER);
 5622 %}
 5623 
 5624 operand vRegD_V27()
 5625 %{
 5626   constraint(ALLOC_IN_RC(v27_reg));
 5627   match(RegD);
 5628   op_cost(0);
 5629   format %{ %}
 5630   interface(REG_INTER);
 5631 %}
 5632 
 5633 operand vRegD_V28()
 5634 %{
 5635   constraint(ALLOC_IN_RC(v28_reg));
 5636   match(RegD);
 5637   op_cost(0);
 5638   format %{ %}
 5639   interface(REG_INTER);
 5640 %}
 5641 
 5642 operand vRegD_V29()
 5643 %{
 5644   constraint(ALLOC_IN_RC(v29_reg));
 5645   match(RegD);
 5646   op_cost(0);
 5647   format %{ %}
 5648   interface(REG_INTER);
 5649 %}
 5650 
 5651 operand vRegD_V30()
 5652 %{
 5653   constraint(ALLOC_IN_RC(v30_reg));
 5654   match(RegD);
 5655   op_cost(0);
 5656   format %{ %}
 5657   interface(REG_INTER);
 5658 %}
 5659 
 5660 operand vRegD_V31()
 5661 %{
 5662   constraint(ALLOC_IN_RC(v31_reg));
 5663   match(RegD);
 5664   op_cost(0);
 5665   format %{ %}
 5666   interface(REG_INTER);
 5667 %}
 5668 
 5669 operand pReg()
 5670 %{
 5671   constraint(ALLOC_IN_RC(pr_reg));
 5672   match(RegVectMask);
 5673   match(pRegGov);
 5674   op_cost(0);
 5675   format %{ %}
 5676   interface(REG_INTER);
 5677 %}
 5678 
 5679 operand pRegGov()
 5680 %{
 5681   constraint(ALLOC_IN_RC(gov_pr));
 5682   match(RegVectMask);
 5683   op_cost(0);
 5684   format %{ %}
 5685   interface(REG_INTER);
 5686 %}
 5687 
 5688 operand pRegGov_P0()
 5689 %{
 5690   constraint(ALLOC_IN_RC(p0_reg));
 5691   match(RegVectMask);
 5692   op_cost(0);
 5693   format %{ %}
 5694   interface(REG_INTER);
 5695 %}
 5696 
 5697 operand pRegGov_P1()
 5698 %{
 5699   constraint(ALLOC_IN_RC(p1_reg));
 5700   match(RegVectMask);
 5701   op_cost(0);
 5702   format %{ %}
 5703   interface(REG_INTER);
 5704 %}
 5705 
 5706 // Flags register, used as output of signed compare instructions
 5707 
 5708 // note that on AArch64 we also use this register as the output for
 5709 // for floating point compare instructions (CmpF CmpD). this ensures
 5710 // that ordered inequality tests use GT, GE, LT or LE none of which
 5711 // pass through cases where the result is unordered i.e. one or both
 5712 // inputs to the compare is a NaN. this means that the ideal code can
 5713 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5714 // (where the comparison should always fail). EQ and NE tests are
 5715 // always generated in ideal code so that unordered folds into the NE
 5716 // case, matching the behaviour of AArch64 NE.
 5717 //
 5718 // This differs from x86 where the outputs of FP compares use a
 5719 // special FP flags registers and where compares based on this
 5720 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5721 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5722 // to explicitly handle the unordered case in branches. x86 also has
 5723 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5724 
 5725 operand rFlagsReg()
 5726 %{
 5727   constraint(ALLOC_IN_RC(int_flags));
 5728   match(RegFlags);
 5729 
 5730   op_cost(0);
 5731   format %{ "RFLAGS" %}
 5732   interface(REG_INTER);
 5733 %}
 5734 
 5735 // Flags register, used as output of unsigned compare instructions
 5736 operand rFlagsRegU()
 5737 %{
 5738   constraint(ALLOC_IN_RC(int_flags));
 5739   match(RegFlags);
 5740 
 5741   op_cost(0);
 5742   format %{ "RFLAGSU" %}
 5743   interface(REG_INTER);
 5744 %}
 5745 
 5746 // Special Registers
 5747 
 5748 // Method Register
 5749 operand inline_cache_RegP(iRegP reg)
 5750 %{
 5751   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5752   match(reg);
 5753   match(iRegPNoSp);
 5754   op_cost(0);
 5755   format %{ %}
 5756   interface(REG_INTER);
 5757 %}
 5758 
 5759 // Thread Register
 5760 operand thread_RegP(iRegP reg)
 5761 %{
 5762   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5763   match(reg);
 5764   op_cost(0);
 5765   format %{ %}
 5766   interface(REG_INTER);
 5767 %}
 5768 
 5769 operand lr_RegP(iRegP reg)
 5770 %{
 5771   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5772   match(reg);
 5773   op_cost(0);
 5774   format %{ %}
 5775   interface(REG_INTER);
 5776 %}
 5777 
 5778 //----------Memory Operands----------------------------------------------------
 5779 
 5780 operand indirect(iRegP reg)
 5781 %{
 5782   constraint(ALLOC_IN_RC(ptr_reg));
 5783   match(reg);
 5784   op_cost(0);
 5785   format %{ "[$reg]" %}
 5786   interface(MEMORY_INTER) %{
 5787     base($reg);
 5788     index(0xffffffff);
 5789     scale(0x0);
 5790     disp(0x0);
 5791   %}
 5792 %}
 5793 
 5794 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5795 %{
 5796   constraint(ALLOC_IN_RC(ptr_reg));
 5797   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5798   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5799   op_cost(0);
 5800   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5801   interface(MEMORY_INTER) %{
 5802     base($reg);
 5803     index($ireg);
 5804     scale($scale);
 5805     disp(0x0);
 5806   %}
 5807 %}
 5808 
 5809 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5810 %{
 5811   constraint(ALLOC_IN_RC(ptr_reg));
 5812   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5813   match(AddP reg (LShiftL lreg scale));
 5814   op_cost(0);
 5815   format %{ "$reg, $lreg lsl($scale)" %}
 5816   interface(MEMORY_INTER) %{
 5817     base($reg);
 5818     index($lreg);
 5819     scale($scale);
 5820     disp(0x0);
 5821   %}
 5822 %}
 5823 
 5824 operand indIndexI2L(iRegP reg, iRegI ireg)
 5825 %{
 5826   constraint(ALLOC_IN_RC(ptr_reg));
 5827   match(AddP reg (ConvI2L ireg));
 5828   op_cost(0);
 5829   format %{ "$reg, $ireg, 0, I2L" %}
 5830   interface(MEMORY_INTER) %{
 5831     base($reg);
 5832     index($ireg);
 5833     scale(0x0);
 5834     disp(0x0);
 5835   %}
 5836 %}
 5837 
 5838 operand indIndex(iRegP reg, iRegL lreg)
 5839 %{
 5840   constraint(ALLOC_IN_RC(ptr_reg));
 5841   match(AddP reg lreg);
 5842   op_cost(0);
 5843   format %{ "$reg, $lreg" %}
 5844   interface(MEMORY_INTER) %{
 5845     base($reg);
 5846     index($lreg);
 5847     scale(0x0);
 5848     disp(0x0);
 5849   %}
 5850 %}
 5851 
 5852 operand indOffI(iRegP reg, immIOffset off)
 5853 %{
 5854   constraint(ALLOC_IN_RC(ptr_reg));
 5855   match(AddP reg off);
 5856   op_cost(0);
 5857   format %{ "[$reg, $off]" %}
 5858   interface(MEMORY_INTER) %{
 5859     base($reg);
 5860     index(0xffffffff);
 5861     scale(0x0);
 5862     disp($off);
 5863   %}
 5864 %}
 5865 
 5866 operand indOffI1(iRegP reg, immIOffset1 off)
 5867 %{
 5868   constraint(ALLOC_IN_RC(ptr_reg));
 5869   match(AddP reg off);
 5870   op_cost(0);
 5871   format %{ "[$reg, $off]" %}
 5872   interface(MEMORY_INTER) %{
 5873     base($reg);
 5874     index(0xffffffff);
 5875     scale(0x0);
 5876     disp($off);
 5877   %}
 5878 %}
 5879 
 5880 operand indOffI2(iRegP reg, immIOffset2 off)
 5881 %{
 5882   constraint(ALLOC_IN_RC(ptr_reg));
 5883   match(AddP reg off);
 5884   op_cost(0);
 5885   format %{ "[$reg, $off]" %}
 5886   interface(MEMORY_INTER) %{
 5887     base($reg);
 5888     index(0xffffffff);
 5889     scale(0x0);
 5890     disp($off);
 5891   %}
 5892 %}
 5893 
 5894 operand indOffI4(iRegP reg, immIOffset4 off)
 5895 %{
 5896   constraint(ALLOC_IN_RC(ptr_reg));
 5897   match(AddP reg off);
 5898   op_cost(0);
 5899   format %{ "[$reg, $off]" %}
 5900   interface(MEMORY_INTER) %{
 5901     base($reg);
 5902     index(0xffffffff);
 5903     scale(0x0);
 5904     disp($off);
 5905   %}
 5906 %}
 5907 
 5908 operand indOffI8(iRegP reg, immIOffset8 off)
 5909 %{
 5910   constraint(ALLOC_IN_RC(ptr_reg));
 5911   match(AddP reg off);
 5912   op_cost(0);
 5913   format %{ "[$reg, $off]" %}
 5914   interface(MEMORY_INTER) %{
 5915     base($reg);
 5916     index(0xffffffff);
 5917     scale(0x0);
 5918     disp($off);
 5919   %}
 5920 %}
 5921 
 5922 operand indOffI16(iRegP reg, immIOffset16 off)
 5923 %{
 5924   constraint(ALLOC_IN_RC(ptr_reg));
 5925   match(AddP reg off);
 5926   op_cost(0);
 5927   format %{ "[$reg, $off]" %}
 5928   interface(MEMORY_INTER) %{
 5929     base($reg);
 5930     index(0xffffffff);
 5931     scale(0x0);
 5932     disp($off);
 5933   %}
 5934 %}
 5935 
 5936 operand indOffL(iRegP reg, immLoffset off)
 5937 %{
 5938   constraint(ALLOC_IN_RC(ptr_reg));
 5939   match(AddP reg off);
 5940   op_cost(0);
 5941   format %{ "[$reg, $off]" %}
 5942   interface(MEMORY_INTER) %{
 5943     base($reg);
 5944     index(0xffffffff);
 5945     scale(0x0);
 5946     disp($off);
 5947   %}
 5948 %}
 5949 
 5950 operand indOffL1(iRegP reg, immLoffset1 off)
 5951 %{
 5952   constraint(ALLOC_IN_RC(ptr_reg));
 5953   match(AddP reg off);
 5954   op_cost(0);
 5955   format %{ "[$reg, $off]" %}
 5956   interface(MEMORY_INTER) %{
 5957     base($reg);
 5958     index(0xffffffff);
 5959     scale(0x0);
 5960     disp($off);
 5961   %}
 5962 %}
 5963 
 5964 operand indOffL2(iRegP reg, immLoffset2 off)
 5965 %{
 5966   constraint(ALLOC_IN_RC(ptr_reg));
 5967   match(AddP reg off);
 5968   op_cost(0);
 5969   format %{ "[$reg, $off]" %}
 5970   interface(MEMORY_INTER) %{
 5971     base($reg);
 5972     index(0xffffffff);
 5973     scale(0x0);
 5974     disp($off);
 5975   %}
 5976 %}
 5977 
 5978 operand indOffL4(iRegP reg, immLoffset4 off)
 5979 %{
 5980   constraint(ALLOC_IN_RC(ptr_reg));
 5981   match(AddP reg off);
 5982   op_cost(0);
 5983   format %{ "[$reg, $off]" %}
 5984   interface(MEMORY_INTER) %{
 5985     base($reg);
 5986     index(0xffffffff);
 5987     scale(0x0);
 5988     disp($off);
 5989   %}
 5990 %}
 5991 
 5992 operand indOffL8(iRegP reg, immLoffset8 off)
 5993 %{
 5994   constraint(ALLOC_IN_RC(ptr_reg));
 5995   match(AddP reg off);
 5996   op_cost(0);
 5997   format %{ "[$reg, $off]" %}
 5998   interface(MEMORY_INTER) %{
 5999     base($reg);
 6000     index(0xffffffff);
 6001     scale(0x0);
 6002     disp($off);
 6003   %}
 6004 %}
 6005 
 6006 operand indOffL16(iRegP reg, immLoffset16 off)
 6007 %{
 6008   constraint(ALLOC_IN_RC(ptr_reg));
 6009   match(AddP reg off);
 6010   op_cost(0);
 6011   format %{ "[$reg, $off]" %}
 6012   interface(MEMORY_INTER) %{
 6013     base($reg);
 6014     index(0xffffffff);
 6015     scale(0x0);
 6016     disp($off);
 6017   %}
 6018 %}
 6019 
 6020 operand indirectN(iRegN reg)
 6021 %{
 6022   predicate(CompressedOops::shift() == 0);
 6023   constraint(ALLOC_IN_RC(ptr_reg));
 6024   match(DecodeN reg);
 6025   op_cost(0);
 6026   format %{ "[$reg]\t# narrow" %}
 6027   interface(MEMORY_INTER) %{
 6028     base($reg);
 6029     index(0xffffffff);
 6030     scale(0x0);
 6031     disp(0x0);
 6032   %}
 6033 %}
 6034 
 6035 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 6036 %{
 6037   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6038   constraint(ALLOC_IN_RC(ptr_reg));
 6039   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 6040   op_cost(0);
 6041   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 6042   interface(MEMORY_INTER) %{
 6043     base($reg);
 6044     index($ireg);
 6045     scale($scale);
 6046     disp(0x0);
 6047   %}
 6048 %}
 6049 
 6050 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 6051 %{
 6052   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6053   constraint(ALLOC_IN_RC(ptr_reg));
 6054   match(AddP (DecodeN reg) (LShiftL lreg scale));
 6055   op_cost(0);
 6056   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 6057   interface(MEMORY_INTER) %{
 6058     base($reg);
 6059     index($lreg);
 6060     scale($scale);
 6061     disp(0x0);
 6062   %}
 6063 %}
 6064 
 6065 operand indIndexI2LN(iRegN reg, iRegI ireg)
 6066 %{
 6067   predicate(CompressedOops::shift() == 0);
 6068   constraint(ALLOC_IN_RC(ptr_reg));
 6069   match(AddP (DecodeN reg) (ConvI2L ireg));
 6070   op_cost(0);
 6071   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 6072   interface(MEMORY_INTER) %{
 6073     base($reg);
 6074     index($ireg);
 6075     scale(0x0);
 6076     disp(0x0);
 6077   %}
 6078 %}
 6079 
 6080 operand indIndexN(iRegN reg, iRegL lreg)
 6081 %{
 6082   predicate(CompressedOops::shift() == 0);
 6083   constraint(ALLOC_IN_RC(ptr_reg));
 6084   match(AddP (DecodeN reg) lreg);
 6085   op_cost(0);
 6086   format %{ "$reg, $lreg\t# narrow" %}
 6087   interface(MEMORY_INTER) %{
 6088     base($reg);
 6089     index($lreg);
 6090     scale(0x0);
 6091     disp(0x0);
 6092   %}
 6093 %}
 6094 
 6095 operand indOffIN(iRegN reg, immIOffset off)
 6096 %{
 6097   predicate(CompressedOops::shift() == 0);
 6098   constraint(ALLOC_IN_RC(ptr_reg));
 6099   match(AddP (DecodeN reg) off);
 6100   op_cost(0);
 6101   format %{ "[$reg, $off]\t# narrow" %}
 6102   interface(MEMORY_INTER) %{
 6103     base($reg);
 6104     index(0xffffffff);
 6105     scale(0x0);
 6106     disp($off);
 6107   %}
 6108 %}
 6109 
 6110 operand indOffLN(iRegN reg, immLoffset off)
 6111 %{
 6112   predicate(CompressedOops::shift() == 0);
 6113   constraint(ALLOC_IN_RC(ptr_reg));
 6114   match(AddP (DecodeN reg) off);
 6115   op_cost(0);
 6116   format %{ "[$reg, $off]\t# narrow" %}
 6117   interface(MEMORY_INTER) %{
 6118     base($reg);
 6119     index(0xffffffff);
 6120     scale(0x0);
 6121     disp($off);
 6122   %}
 6123 %}
 6124 
 6125 
 6126 
 6127 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6128 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6129 %{
 6130   constraint(ALLOC_IN_RC(ptr_reg));
 6131   match(AddP reg off);
 6132   op_cost(0);
 6133   format %{ "[$reg, $off]" %}
 6134   interface(MEMORY_INTER) %{
 6135     base($reg);
 6136     index(0xffffffff);
 6137     scale(0x0);
 6138     disp($off);
 6139   %}
 6140 %}
 6141 
 6142 //----------Special Memory Operands--------------------------------------------
 6143 // Stack Slot Operand - This operand is used for loading and storing temporary
 6144 //                      values on the stack where a match requires a value to
 6145 //                      flow through memory.
 6146 operand stackSlotP(sRegP reg)
 6147 %{
 6148   constraint(ALLOC_IN_RC(stack_slots));
 6149   op_cost(100);
 6150   // No match rule because this operand is only generated in matching
 6151   // match(RegP);
 6152   format %{ "[$reg]" %}
 6153   interface(MEMORY_INTER) %{
 6154     base(0x1e);  // RSP
 6155     index(0x0);  // No Index
 6156     scale(0x0);  // No Scale
 6157     disp($reg);  // Stack Offset
 6158   %}
 6159 %}
 6160 
 6161 operand stackSlotI(sRegI reg)
 6162 %{
 6163   constraint(ALLOC_IN_RC(stack_slots));
 6164   // No match rule because this operand is only generated in matching
 6165   // match(RegI);
 6166   format %{ "[$reg]" %}
 6167   interface(MEMORY_INTER) %{
 6168     base(0x1e);  // RSP
 6169     index(0x0);  // No Index
 6170     scale(0x0);  // No Scale
 6171     disp($reg);  // Stack Offset
 6172   %}
 6173 %}
 6174 
 6175 operand stackSlotF(sRegF reg)
 6176 %{
 6177   constraint(ALLOC_IN_RC(stack_slots));
 6178   // No match rule because this operand is only generated in matching
 6179   // match(RegF);
 6180   format %{ "[$reg]" %}
 6181   interface(MEMORY_INTER) %{
 6182     base(0x1e);  // RSP
 6183     index(0x0);  // No Index
 6184     scale(0x0);  // No Scale
 6185     disp($reg);  // Stack Offset
 6186   %}
 6187 %}
 6188 
 6189 operand stackSlotD(sRegD reg)
 6190 %{
 6191   constraint(ALLOC_IN_RC(stack_slots));
 6192   // No match rule because this operand is only generated in matching
 6193   // match(RegD);
 6194   format %{ "[$reg]" %}
 6195   interface(MEMORY_INTER) %{
 6196     base(0x1e);  // RSP
 6197     index(0x0);  // No Index
 6198     scale(0x0);  // No Scale
 6199     disp($reg);  // Stack Offset
 6200   %}
 6201 %}
 6202 
 6203 operand stackSlotL(sRegL reg)
 6204 %{
 6205   constraint(ALLOC_IN_RC(stack_slots));
 6206   // No match rule because this operand is only generated in matching
 6207   // match(RegL);
 6208   format %{ "[$reg]" %}
 6209   interface(MEMORY_INTER) %{
 6210     base(0x1e);  // RSP
 6211     index(0x0);  // No Index
 6212     scale(0x0);  // No Scale
 6213     disp($reg);  // Stack Offset
 6214   %}
 6215 %}
 6216 
 6217 // Operands for expressing Control Flow
 6218 // NOTE: Label is a predefined operand which should not be redefined in
 6219 //       the AD file. It is generically handled within the ADLC.
 6220 
 6221 //----------Conditional Branch Operands----------------------------------------
 6222 // Comparison Op  - This is the operation of the comparison, and is limited to
 6223 //                  the following set of codes:
 6224 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6225 //
 6226 // Other attributes of the comparison, such as unsignedness, are specified
 6227 // by the comparison instruction that sets a condition code flags register.
 6228 // That result is represented by a flags operand whose subtype is appropriate
 6229 // to the unsignedness (etc.) of the comparison.
 6230 //
 6231 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6232 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6233 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6234 
 6235 // used for signed integral comparisons and fp comparisons
 6236 
 6237 operand cmpOp()
 6238 %{
 6239   match(Bool);
 6240 
 6241   format %{ "" %}
 6242   interface(COND_INTER) %{
 6243     equal(0x0, "eq");
 6244     not_equal(0x1, "ne");
 6245     less(0xb, "lt");
 6246     greater_equal(0xa, "ge");
 6247     less_equal(0xd, "le");
 6248     greater(0xc, "gt");
 6249     overflow(0x6, "vs");
 6250     no_overflow(0x7, "vc");
 6251   %}
 6252 %}
 6253 
 6254 // used for unsigned integral comparisons
 6255 
 6256 operand cmpOpU()
 6257 %{
 6258   match(Bool);
 6259 
 6260   format %{ "" %}
 6261   interface(COND_INTER) %{
 6262     equal(0x0, "eq");
 6263     not_equal(0x1, "ne");
 6264     less(0x3, "lo");
 6265     greater_equal(0x2, "hs");
 6266     less_equal(0x9, "ls");
 6267     greater(0x8, "hi");
 6268     overflow(0x6, "vs");
 6269     no_overflow(0x7, "vc");
 6270   %}
 6271 %}
 6272 
 6273 // used for certain integral comparisons which can be
 6274 // converted to cbxx or tbxx instructions
 6275 
 6276 operand cmpOpEqNe()
 6277 %{
 6278   match(Bool);
 6279   op_cost(0);
 6280   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6281             || n->as_Bool()->_test._test == BoolTest::eq);
 6282 
 6283   format %{ "" %}
 6284   interface(COND_INTER) %{
 6285     equal(0x0, "eq");
 6286     not_equal(0x1, "ne");
 6287     less(0xb, "lt");
 6288     greater_equal(0xa, "ge");
 6289     less_equal(0xd, "le");
 6290     greater(0xc, "gt");
 6291     overflow(0x6, "vs");
 6292     no_overflow(0x7, "vc");
 6293   %}
 6294 %}
 6295 
 6296 // used for certain integral comparisons which can be
 6297 // converted to cbxx or tbxx instructions
 6298 
 6299 operand cmpOpLtGe()
 6300 %{
 6301   match(Bool);
 6302   op_cost(0);
 6303 
 6304   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6305             || n->as_Bool()->_test._test == BoolTest::ge);
 6306 
 6307   format %{ "" %}
 6308   interface(COND_INTER) %{
 6309     equal(0x0, "eq");
 6310     not_equal(0x1, "ne");
 6311     less(0xb, "lt");
 6312     greater_equal(0xa, "ge");
 6313     less_equal(0xd, "le");
 6314     greater(0xc, "gt");
 6315     overflow(0x6, "vs");
 6316     no_overflow(0x7, "vc");
 6317   %}
 6318 %}
 6319 
 6320 // used for certain unsigned integral comparisons which can be
 6321 // converted to cbxx or tbxx instructions
 6322 
 6323 operand cmpOpUEqNeLtGe()
 6324 %{
 6325   match(Bool);
 6326   op_cost(0);
 6327 
 6328   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6329             || n->as_Bool()->_test._test == BoolTest::ne
 6330             || n->as_Bool()->_test._test == BoolTest::lt
 6331             || n->as_Bool()->_test._test == BoolTest::ge);
 6332 
 6333   format %{ "" %}
 6334   interface(COND_INTER) %{
 6335     equal(0x0, "eq");
 6336     not_equal(0x1, "ne");
 6337     less(0xb, "lt");
 6338     greater_equal(0xa, "ge");
 6339     less_equal(0xd, "le");
 6340     greater(0xc, "gt");
 6341     overflow(0x6, "vs");
 6342     no_overflow(0x7, "vc");
 6343   %}
 6344 %}
 6345 
 6346 // Special operand allowing long args to int ops to be truncated for free
 6347 
 6348 operand iRegL2I(iRegL reg) %{
 6349 
 6350   op_cost(0);
 6351 
 6352   match(ConvL2I reg);
 6353 
 6354   format %{ "l2i($reg)" %}
 6355 
 6356   interface(REG_INTER)
 6357 %}
 6358 
 6359 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6360 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6361 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6362 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6363 
 6364 //----------OPERAND CLASSES----------------------------------------------------
 6365 // Operand Classes are groups of operands that are used as to simplify
 6366 // instruction definitions by not requiring the AD writer to specify
 6367 // separate instructions for every form of operand when the
 6368 // instruction accepts multiple operand types with the same basic
 6369 // encoding and format. The classic case of this is memory operands.
 6370 
 6371 // memory is used to define read/write location for load/store
 6372 // instruction defs. we can turn a memory op into an Address
 6373 
 6374 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6375                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6376 
 6377 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6378                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6379 
 6380 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6381                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6382 
 6383 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6384                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6385 
 6386 // All of the memory operands. For the pipeline description.
 6387 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6388                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6389                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6390 
 6391 
 6392 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6393 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6394 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6395 // can be elided because the 32-bit instruction will just employ the
 6396 // lower 32 bits anyway.
 6397 //
 6398 // n.b. this does not elide all L2I conversions. if the truncated
 6399 // value is consumed by more than one operation then the ConvL2I
 6400 // cannot be bundled into the consuming nodes so an l2i gets planted
 6401 // (actually a movw $dst $src) and the downstream instructions consume
 6402 // the result of the l2i as an iRegI input. That's a shame since the
 6403 // movw is actually redundant but its not too costly.
 6404 
 6405 opclass iRegIorL2I(iRegI, iRegL2I);
 6406 
 6407 //----------PIPELINE-----------------------------------------------------------
 6408 // Rules which define the behavior of the target architectures pipeline.
 6409 
 6410 // For specific pipelines, eg A53, define the stages of that pipeline
 6411 //pipe_desc(ISS, EX1, EX2, WR);
 6412 #define ISS S0
 6413 #define EX1 S1
 6414 #define EX2 S2
 6415 #define WR  S3
 6416 
 6417 // Integer ALU reg operation
 6418 pipeline %{
 6419 
 6420 attributes %{
 6421   // ARM instructions are of fixed length
 6422   fixed_size_instructions;        // Fixed size instructions TODO does
 6423   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6424   // ARM instructions come in 32-bit word units
 6425   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6426   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6427   instruction_fetch_units = 1;       // of 64 bytes
 6428 
 6429   // List of nop instructions
 6430   nops( MachNop );
 6431 %}
 6432 
 6433 // We don't use an actual pipeline model so don't care about resources
 6434 // or description. we do use pipeline classes to introduce fixed
 6435 // latencies
 6436 
 6437 //----------RESOURCES----------------------------------------------------------
 6438 // Resources are the functional units available to the machine
 6439 
 6440 resources( INS0, INS1, INS01 = INS0 | INS1,
 6441            ALU0, ALU1, ALU = ALU0 | ALU1,
 6442            MAC,
 6443            DIV,
 6444            BRANCH,
 6445            LDST,
 6446            NEON_FP);
 6447 
 6448 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6449 // Pipeline Description specifies the stages in the machine's pipeline
 6450 
 6451 // Define the pipeline as a generic 6 stage pipeline
 6452 pipe_desc(S0, S1, S2, S3, S4, S5);
 6453 
 6454 //----------PIPELINE CLASSES---------------------------------------------------
 6455 // Pipeline Classes describe the stages in which input and output are
 6456 // referenced by the hardware pipeline.
 6457 
 6458 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6459 %{
 6460   single_instruction;
 6461   src1   : S1(read);
 6462   src2   : S2(read);
 6463   dst    : S5(write);
 6464   INS01  : ISS;
 6465   NEON_FP : S5;
 6466 %}
 6467 
 6468 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6469 %{
 6470   single_instruction;
 6471   src1   : S1(read);
 6472   src2   : S2(read);
 6473   dst    : S5(write);
 6474   INS01  : ISS;
 6475   NEON_FP : S5;
 6476 %}
 6477 
 6478 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6479 %{
 6480   single_instruction;
 6481   src    : S1(read);
 6482   dst    : S5(write);
 6483   INS01  : ISS;
 6484   NEON_FP : S5;
 6485 %}
 6486 
 6487 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6488 %{
 6489   single_instruction;
 6490   src    : S1(read);
 6491   dst    : S5(write);
 6492   INS01  : ISS;
 6493   NEON_FP : S5;
 6494 %}
 6495 
 6496 pipe_class fp_d2f(vRegF dst, vRegD src)
 6497 %{
 6498   single_instruction;
 6499   src    : S1(read);
 6500   dst    : S5(write);
 6501   INS01  : ISS;
 6502   NEON_FP : S5;
 6503 %}
 6504 
 6505 pipe_class fp_f2d(vRegD dst, vRegF src)
 6506 %{
 6507   single_instruction;
 6508   src    : S1(read);
 6509   dst    : S5(write);
 6510   INS01  : ISS;
 6511   NEON_FP : S5;
 6512 %}
 6513 
 6514 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6515 %{
 6516   single_instruction;
 6517   src    : S1(read);
 6518   dst    : S5(write);
 6519   INS01  : ISS;
 6520   NEON_FP : S5;
 6521 %}
 6522 
 6523 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6524 %{
 6525   single_instruction;
 6526   src    : S1(read);
 6527   dst    : S5(write);
 6528   INS01  : ISS;
 6529   NEON_FP : S5;
 6530 %}
 6531 
 6532 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6533 %{
 6534   single_instruction;
 6535   src    : S1(read);
 6536   dst    : S5(write);
 6537   INS01  : ISS;
 6538   NEON_FP : S5;
 6539 %}
 6540 
 6541 pipe_class fp_l2f(vRegF dst, iRegL src)
 6542 %{
 6543   single_instruction;
 6544   src    : S1(read);
 6545   dst    : S5(write);
 6546   INS01  : ISS;
 6547   NEON_FP : S5;
 6548 %}
 6549 
 6550 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6551 %{
 6552   single_instruction;
 6553   src    : S1(read);
 6554   dst    : S5(write);
 6555   INS01  : ISS;
 6556   NEON_FP : S5;
 6557 %}
 6558 
 6559 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6560 %{
 6561   single_instruction;
 6562   src    : S1(read);
 6563   dst    : S5(write);
 6564   INS01  : ISS;
 6565   NEON_FP : S5;
 6566 %}
 6567 
 6568 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6569 %{
 6570   single_instruction;
 6571   src    : S1(read);
 6572   dst    : S5(write);
 6573   INS01  : ISS;
 6574   NEON_FP : S5;
 6575 %}
 6576 
 6577 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6578 %{
 6579   single_instruction;
 6580   src    : S1(read);
 6581   dst    : S5(write);
 6582   INS01  : ISS;
 6583   NEON_FP : S5;
 6584 %}
 6585 
 6586 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6587 %{
 6588   single_instruction;
 6589   src1   : S1(read);
 6590   src2   : S2(read);
 6591   dst    : S5(write);
 6592   INS0   : ISS;
 6593   NEON_FP : S5;
 6594 %}
 6595 
 6596 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6597 %{
 6598   single_instruction;
 6599   src1   : S1(read);
 6600   src2   : S2(read);
 6601   dst    : S5(write);
 6602   INS0   : ISS;
 6603   NEON_FP : S5;
 6604 %}
 6605 
 6606 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6607 %{
 6608   single_instruction;
 6609   cr     : S1(read);
 6610   src1   : S1(read);
 6611   src2   : S1(read);
 6612   dst    : S3(write);
 6613   INS01  : ISS;
 6614   NEON_FP : S3;
 6615 %}
 6616 
 6617 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6618 %{
 6619   single_instruction;
 6620   cr     : S1(read);
 6621   src1   : S1(read);
 6622   src2   : S1(read);
 6623   dst    : S3(write);
 6624   INS01  : ISS;
 6625   NEON_FP : S3;
 6626 %}
 6627 
 6628 pipe_class fp_imm_s(vRegF dst)
 6629 %{
 6630   single_instruction;
 6631   dst    : S3(write);
 6632   INS01  : ISS;
 6633   NEON_FP : S3;
 6634 %}
 6635 
 6636 pipe_class fp_imm_d(vRegD dst)
 6637 %{
 6638   single_instruction;
 6639   dst    : S3(write);
 6640   INS01  : ISS;
 6641   NEON_FP : S3;
 6642 %}
 6643 
 6644 pipe_class fp_load_constant_s(vRegF dst)
 6645 %{
 6646   single_instruction;
 6647   dst    : S4(write);
 6648   INS01  : ISS;
 6649   NEON_FP : S4;
 6650 %}
 6651 
 6652 pipe_class fp_load_constant_d(vRegD dst)
 6653 %{
 6654   single_instruction;
 6655   dst    : S4(write);
 6656   INS01  : ISS;
 6657   NEON_FP : S4;
 6658 %}
 6659 
 6660 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6661 %{
 6662   single_instruction;
 6663   dst    : S5(write);
 6664   src1   : S1(read);
 6665   src2   : S1(read);
 6666   INS01  : ISS;
 6667   NEON_FP : S5;
 6668 %}
 6669 
 6670 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6671 %{
 6672   single_instruction;
 6673   dst    : S5(write);
 6674   src1   : S1(read);
 6675   src2   : S1(read);
 6676   INS0   : ISS;
 6677   NEON_FP : S5;
 6678 %}
 6679 
 6680 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6681 %{
 6682   single_instruction;
 6683   dst    : S5(write);
 6684   src1   : S1(read);
 6685   src2   : S1(read);
 6686   dst    : S1(read);
 6687   INS01  : ISS;
 6688   NEON_FP : S5;
 6689 %}
 6690 
 6691 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6692 %{
 6693   single_instruction;
 6694   dst    : S5(write);
 6695   src1   : S1(read);
 6696   src2   : S1(read);
 6697   dst    : S1(read);
 6698   INS0   : ISS;
 6699   NEON_FP : S5;
 6700 %}
 6701 
 6702 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6703 %{
 6704   single_instruction;
 6705   dst    : S4(write);
 6706   src1   : S2(read);
 6707   src2   : S2(read);
 6708   INS01  : ISS;
 6709   NEON_FP : S4;
 6710 %}
 6711 
 6712 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6713 %{
 6714   single_instruction;
 6715   dst    : S4(write);
 6716   src1   : S2(read);
 6717   src2   : S2(read);
 6718   INS0   : ISS;
 6719   NEON_FP : S4;
 6720 %}
 6721 
 6722 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6723 %{
 6724   single_instruction;
 6725   dst    : S3(write);
 6726   src1   : S2(read);
 6727   src2   : S2(read);
 6728   INS01  : ISS;
 6729   NEON_FP : S3;
 6730 %}
 6731 
 6732 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6733 %{
 6734   single_instruction;
 6735   dst    : S3(write);
 6736   src1   : S2(read);
 6737   src2   : S2(read);
 6738   INS0   : ISS;
 6739   NEON_FP : S3;
 6740 %}
 6741 
 6742 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6743 %{
 6744   single_instruction;
 6745   dst    : S3(write);
 6746   src    : S1(read);
 6747   shift  : S1(read);
 6748   INS01  : ISS;
 6749   NEON_FP : S3;
 6750 %}
 6751 
 6752 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6753 %{
 6754   single_instruction;
 6755   dst    : S3(write);
 6756   src    : S1(read);
 6757   shift  : S1(read);
 6758   INS0   : ISS;
 6759   NEON_FP : S3;
 6760 %}
 6761 
 6762 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6763 %{
 6764   single_instruction;
 6765   dst    : S3(write);
 6766   src    : S1(read);
 6767   INS01  : ISS;
 6768   NEON_FP : S3;
 6769 %}
 6770 
 6771 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6772 %{
 6773   single_instruction;
 6774   dst    : S3(write);
 6775   src    : S1(read);
 6776   INS0   : ISS;
 6777   NEON_FP : S3;
 6778 %}
 6779 
 6780 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6781 %{
 6782   single_instruction;
 6783   dst    : S5(write);
 6784   src1   : S1(read);
 6785   src2   : S1(read);
 6786   INS01  : ISS;
 6787   NEON_FP : S5;
 6788 %}
 6789 
 6790 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6791 %{
 6792   single_instruction;
 6793   dst    : S5(write);
 6794   src1   : S1(read);
 6795   src2   : S1(read);
 6796   INS0   : ISS;
 6797   NEON_FP : S5;
 6798 %}
 6799 
 6800 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6801 %{
 6802   single_instruction;
 6803   dst    : S5(write);
 6804   src1   : S1(read);
 6805   src2   : S1(read);
 6806   INS0   : ISS;
 6807   NEON_FP : S5;
 6808 %}
 6809 
 6810 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6811 %{
 6812   single_instruction;
 6813   dst    : S5(write);
 6814   src1   : S1(read);
 6815   src2   : S1(read);
 6816   INS0   : ISS;
 6817   NEON_FP : S5;
 6818 %}
 6819 
 6820 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6821 %{
 6822   single_instruction;
 6823   dst    : S5(write);
 6824   src    : S1(read);
 6825   INS0   : ISS;
 6826   NEON_FP : S5;
 6827 %}
 6828 
 6829 pipe_class vunop_fp64(vecD dst, vecD src)
 6830 %{
 6831   single_instruction;
 6832   dst    : S5(write);
 6833   src    : S1(read);
 6834   INS01  : ISS;
 6835   NEON_FP : S5;
 6836 %}
 6837 
 6838 pipe_class vunop_fp128(vecX dst, vecX src)
 6839 %{
 6840   single_instruction;
 6841   dst    : S5(write);
 6842   src    : S1(read);
 6843   INS0   : ISS;
 6844   NEON_FP : S5;
 6845 %}
 6846 
 6847 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6848 %{
 6849   single_instruction;
 6850   dst    : S3(write);
 6851   src    : S1(read);
 6852   INS01  : ISS;
 6853   NEON_FP : S3;
 6854 %}
 6855 
 6856 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6857 %{
 6858   single_instruction;
 6859   dst    : S3(write);
 6860   src    : S1(read);
 6861   INS01  : ISS;
 6862   NEON_FP : S3;
 6863 %}
 6864 
 6865 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6866 %{
 6867   single_instruction;
 6868   dst    : S3(write);
 6869   src    : S1(read);
 6870   INS01  : ISS;
 6871   NEON_FP : S3;
 6872 %}
 6873 
 6874 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6875 %{
 6876   single_instruction;
 6877   dst    : S3(write);
 6878   src    : S1(read);
 6879   INS01  : ISS;
 6880   NEON_FP : S3;
 6881 %}
 6882 
 6883 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6884 %{
 6885   single_instruction;
 6886   dst    : S3(write);
 6887   src    : S1(read);
 6888   INS01  : ISS;
 6889   NEON_FP : S3;
 6890 %}
 6891 
 6892 pipe_class vmovi_reg_imm64(vecD dst)
 6893 %{
 6894   single_instruction;
 6895   dst    : S3(write);
 6896   INS01  : ISS;
 6897   NEON_FP : S3;
 6898 %}
 6899 
 6900 pipe_class vmovi_reg_imm128(vecX dst)
 6901 %{
 6902   single_instruction;
 6903   dst    : S3(write);
 6904   INS0   : ISS;
 6905   NEON_FP : S3;
 6906 %}
 6907 
 6908 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6909 %{
 6910   single_instruction;
 6911   dst    : S5(write);
 6912   mem    : ISS(read);
 6913   INS01  : ISS;
 6914   NEON_FP : S3;
 6915 %}
 6916 
 6917 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6918 %{
 6919   single_instruction;
 6920   dst    : S5(write);
 6921   mem    : ISS(read);
 6922   INS01  : ISS;
 6923   NEON_FP : S3;
 6924 %}
 6925 
 6926 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6927 %{
 6928   single_instruction;
 6929   mem    : ISS(read);
 6930   src    : S2(read);
 6931   INS01  : ISS;
 6932   NEON_FP : S3;
 6933 %}
 6934 
 6935 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6936 %{
 6937   single_instruction;
 6938   mem    : ISS(read);
 6939   src    : S2(read);
 6940   INS01  : ISS;
 6941   NEON_FP : S3;
 6942 %}
 6943 
 6944 //------- Integer ALU operations --------------------------
 6945 
 6946 // Integer ALU reg-reg operation
 6947 // Operands needed in EX1, result generated in EX2
 6948 // Eg.  ADD     x0, x1, x2
 6949 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6950 %{
 6951   single_instruction;
 6952   dst    : EX2(write);
 6953   src1   : EX1(read);
 6954   src2   : EX1(read);
 6955   INS01  : ISS; // Dual issue as instruction 0 or 1
 6956   ALU    : EX2;
 6957 %}
 6958 
 6959 // Integer ALU reg-reg operation with constant shift
 6960 // Shifted register must be available in LATE_ISS instead of EX1
 6961 // Eg.  ADD     x0, x1, x2, LSL #2
 6962 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6963 %{
 6964   single_instruction;
 6965   dst    : EX2(write);
 6966   src1   : EX1(read);
 6967   src2   : ISS(read);
 6968   INS01  : ISS;
 6969   ALU    : EX2;
 6970 %}
 6971 
 6972 // Integer ALU reg operation with constant shift
 6973 // Eg.  LSL     x0, x1, #shift
 6974 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6975 %{
 6976   single_instruction;
 6977   dst    : EX2(write);
 6978   src1   : ISS(read);
 6979   INS01  : ISS;
 6980   ALU    : EX2;
 6981 %}
 6982 
 6983 // Integer ALU reg-reg operation with variable shift
 6984 // Both operands must be available in LATE_ISS instead of EX1
 6985 // Result is available in EX1 instead of EX2
 6986 // Eg.  LSLV    x0, x1, x2
 6987 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6988 %{
 6989   single_instruction;
 6990   dst    : EX1(write);
 6991   src1   : ISS(read);
 6992   src2   : ISS(read);
 6993   INS01  : ISS;
 6994   ALU    : EX1;
 6995 %}
 6996 
 6997 // Integer ALU reg-reg operation with extract
 6998 // As for _vshift above, but result generated in EX2
 6999 // Eg.  EXTR    x0, x1, x2, #N
 7000 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 7001 %{
 7002   single_instruction;
 7003   dst    : EX2(write);
 7004   src1   : ISS(read);
 7005   src2   : ISS(read);
 7006   INS1   : ISS; // Can only dual issue as Instruction 1
 7007   ALU    : EX1;
 7008 %}
 7009 
 7010 // Integer ALU reg operation
 7011 // Eg.  NEG     x0, x1
 7012 pipe_class ialu_reg(iRegI dst, iRegI src)
 7013 %{
 7014   single_instruction;
 7015   dst    : EX2(write);
 7016   src    : EX1(read);
 7017   INS01  : ISS;
 7018   ALU    : EX2;
 7019 %}
 7020 
 7021 // Integer ALU reg mmediate operation
 7022 // Eg.  ADD     x0, x1, #N
 7023 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 7024 %{
 7025   single_instruction;
 7026   dst    : EX2(write);
 7027   src1   : EX1(read);
 7028   INS01  : ISS;
 7029   ALU    : EX2;
 7030 %}
 7031 
 7032 // Integer ALU immediate operation (no source operands)
 7033 // Eg.  MOV     x0, #N
 7034 pipe_class ialu_imm(iRegI dst)
 7035 %{
 7036   single_instruction;
 7037   dst    : EX1(write);
 7038   INS01  : ISS;
 7039   ALU    : EX1;
 7040 %}
 7041 
 7042 //------- Compare operation -------------------------------
 7043 
 7044 // Compare reg-reg
 7045 // Eg.  CMP     x0, x1
 7046 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 7047 %{
 7048   single_instruction;
 7049 //  fixed_latency(16);
 7050   cr     : EX2(write);
 7051   op1    : EX1(read);
 7052   op2    : EX1(read);
 7053   INS01  : ISS;
 7054   ALU    : EX2;
 7055 %}
 7056 
 7057 // Compare reg-reg
 7058 // Eg.  CMP     x0, #N
 7059 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 7060 %{
 7061   single_instruction;
 7062 //  fixed_latency(16);
 7063   cr     : EX2(write);
 7064   op1    : EX1(read);
 7065   INS01  : ISS;
 7066   ALU    : EX2;
 7067 %}
 7068 
 7069 //------- Conditional instructions ------------------------
 7070 
 7071 // Conditional no operands
 7072 // Eg.  CSINC   x0, zr, zr, <cond>
 7073 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 7074 %{
 7075   single_instruction;
 7076   cr     : EX1(read);
 7077   dst    : EX2(write);
 7078   INS01  : ISS;
 7079   ALU    : EX2;
 7080 %}
 7081 
 7082 // Conditional 2 operand
 7083 // EG.  CSEL    X0, X1, X2, <cond>
 7084 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 7085 %{
 7086   single_instruction;
 7087   cr     : EX1(read);
 7088   src1   : EX1(read);
 7089   src2   : EX1(read);
 7090   dst    : EX2(write);
 7091   INS01  : ISS;
 7092   ALU    : EX2;
 7093 %}
 7094 
 7095 // Conditional 2 operand
 7096 // EG.  CSEL    X0, X1, X2, <cond>
 7097 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 7098 %{
 7099   single_instruction;
 7100   cr     : EX1(read);
 7101   src    : EX1(read);
 7102   dst    : EX2(write);
 7103   INS01  : ISS;
 7104   ALU    : EX2;
 7105 %}
 7106 
 7107 //------- Multiply pipeline operations --------------------
 7108 
 7109 // Multiply reg-reg
 7110 // Eg.  MUL     w0, w1, w2
 7111 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7112 %{
 7113   single_instruction;
 7114   dst    : WR(write);
 7115   src1   : ISS(read);
 7116   src2   : ISS(read);
 7117   INS01  : ISS;
 7118   MAC    : WR;
 7119 %}
 7120 
 7121 // Multiply accumulate
 7122 // Eg.  MADD    w0, w1, w2, w3
 7123 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7124 %{
 7125   single_instruction;
 7126   dst    : WR(write);
 7127   src1   : ISS(read);
 7128   src2   : ISS(read);
 7129   src3   : ISS(read);
 7130   INS01  : ISS;
 7131   MAC    : WR;
 7132 %}
 7133 
 7134 // Eg.  MUL     w0, w1, w2
 7135 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7136 %{
 7137   single_instruction;
 7138   fixed_latency(3); // Maximum latency for 64 bit mul
 7139   dst    : WR(write);
 7140   src1   : ISS(read);
 7141   src2   : ISS(read);
 7142   INS01  : ISS;
 7143   MAC    : WR;
 7144 %}
 7145 
 7146 // Multiply accumulate
 7147 // Eg.  MADD    w0, w1, w2, w3
 7148 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7149 %{
 7150   single_instruction;
 7151   fixed_latency(3); // Maximum latency for 64 bit mul
 7152   dst    : WR(write);
 7153   src1   : ISS(read);
 7154   src2   : ISS(read);
 7155   src3   : ISS(read);
 7156   INS01  : ISS;
 7157   MAC    : WR;
 7158 %}
 7159 
 7160 //------- Divide pipeline operations --------------------
 7161 
 7162 // Eg.  SDIV    w0, w1, w2
 7163 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7164 %{
 7165   single_instruction;
 7166   fixed_latency(8); // Maximum latency for 32 bit divide
 7167   dst    : WR(write);
 7168   src1   : ISS(read);
 7169   src2   : ISS(read);
 7170   INS0   : ISS; // Can only dual issue as instruction 0
 7171   DIV    : WR;
 7172 %}
 7173 
 7174 // Eg.  SDIV    x0, x1, x2
 7175 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7176 %{
 7177   single_instruction;
 7178   fixed_latency(16); // Maximum latency for 64 bit divide
 7179   dst    : WR(write);
 7180   src1   : ISS(read);
 7181   src2   : ISS(read);
 7182   INS0   : ISS; // Can only dual issue as instruction 0
 7183   DIV    : WR;
 7184 %}
 7185 
 7186 //------- Load pipeline operations ------------------------
 7187 
 7188 // Load - prefetch
 7189 // Eg.  PFRM    <mem>
 7190 pipe_class iload_prefetch(memory mem)
 7191 %{
 7192   single_instruction;
 7193   mem    : ISS(read);
 7194   INS01  : ISS;
 7195   LDST   : WR;
 7196 %}
 7197 
 7198 // Load - reg, mem
 7199 // Eg.  LDR     x0, <mem>
 7200 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7201 %{
 7202   single_instruction;
 7203   dst    : WR(write);
 7204   mem    : ISS(read);
 7205   INS01  : ISS;
 7206   LDST   : WR;
 7207 %}
 7208 
 7209 // Load - reg, reg
 7210 // Eg.  LDR     x0, [sp, x1]
 7211 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7212 %{
 7213   single_instruction;
 7214   dst    : WR(write);
 7215   src    : ISS(read);
 7216   INS01  : ISS;
 7217   LDST   : WR;
 7218 %}
 7219 
 7220 //------- Store pipeline operations -----------------------
 7221 
 7222 // Store - zr, mem
 7223 // Eg.  STR     zr, <mem>
 7224 pipe_class istore_mem(memory mem)
 7225 %{
 7226   single_instruction;
 7227   mem    : ISS(read);
 7228   INS01  : ISS;
 7229   LDST   : WR;
 7230 %}
 7231 
 7232 // Store - reg, mem
 7233 // Eg.  STR     x0, <mem>
 7234 pipe_class istore_reg_mem(iRegI src, memory mem)
 7235 %{
 7236   single_instruction;
 7237   mem    : ISS(read);
 7238   src    : EX2(read);
 7239   INS01  : ISS;
 7240   LDST   : WR;
 7241 %}
 7242 
 7243 // Store - reg, reg
 7244 // Eg. STR      x0, [sp, x1]
 7245 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7246 %{
 7247   single_instruction;
 7248   dst    : ISS(read);
 7249   src    : EX2(read);
 7250   INS01  : ISS;
 7251   LDST   : WR;
 7252 %}
 7253 
 7254 //------- Store pipeline operations -----------------------
 7255 
 7256 // Branch
 7257 pipe_class pipe_branch()
 7258 %{
 7259   single_instruction;
 7260   INS01  : ISS;
 7261   BRANCH : EX1;
 7262 %}
 7263 
 7264 // Conditional branch
 7265 pipe_class pipe_branch_cond(rFlagsReg cr)
 7266 %{
 7267   single_instruction;
 7268   cr     : EX1(read);
 7269   INS01  : ISS;
 7270   BRANCH : EX1;
 7271 %}
 7272 
 7273 // Compare & Branch
 7274 // EG.  CBZ/CBNZ
 7275 pipe_class pipe_cmp_branch(iRegI op1)
 7276 %{
 7277   single_instruction;
 7278   op1    : EX1(read);
 7279   INS01  : ISS;
 7280   BRANCH : EX1;
 7281 %}
 7282 
 7283 //------- Synchronisation operations ----------------------
 7284 
 7285 // Any operation requiring serialization.
 7286 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7287 pipe_class pipe_serial()
 7288 %{
 7289   single_instruction;
 7290   force_serialization;
 7291   fixed_latency(16);
 7292   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7293   LDST   : WR;
 7294 %}
 7295 
 7296 // Generic big/slow expanded idiom - also serialized
 7297 pipe_class pipe_slow()
 7298 %{
 7299   instruction_count(10);
 7300   multiple_bundles;
 7301   force_serialization;
 7302   fixed_latency(16);
 7303   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7304   LDST   : WR;
 7305 %}
 7306 
 7307 // Empty pipeline class
 7308 pipe_class pipe_class_empty()
 7309 %{
 7310   single_instruction;
 7311   fixed_latency(0);
 7312 %}
 7313 
 7314 // Default pipeline class.
 7315 pipe_class pipe_class_default()
 7316 %{
 7317   single_instruction;
 7318   fixed_latency(2);
 7319 %}
 7320 
 7321 // Pipeline class for compares.
 7322 pipe_class pipe_class_compare()
 7323 %{
 7324   single_instruction;
 7325   fixed_latency(16);
 7326 %}
 7327 
 7328 // Pipeline class for memory operations.
 7329 pipe_class pipe_class_memory()
 7330 %{
 7331   single_instruction;
 7332   fixed_latency(16);
 7333 %}
 7334 
 7335 // Pipeline class for call.
 7336 pipe_class pipe_class_call()
 7337 %{
 7338   single_instruction;
 7339   fixed_latency(100);
 7340 %}
 7341 
 7342 // Define the class for the Nop node.
 7343 define %{
 7344    MachNop = pipe_class_empty;
 7345 %}
 7346 
 7347 %}
 7348 //----------INSTRUCTIONS-------------------------------------------------------
 7349 //
 7350 // match      -- States which machine-independent subtree may be replaced
 7351 //               by this instruction.
 7352 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7353 //               selection to identify a minimum cost tree of machine
 7354 //               instructions that matches a tree of machine-independent
 7355 //               instructions.
 7356 // format     -- A string providing the disassembly for this instruction.
 7357 //               The value of an instruction's operand may be inserted
 7358 //               by referring to it with a '$' prefix.
 7359 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7360 //               to within an encode class as $primary, $secondary, and $tertiary
 7361 //               rrspectively.  The primary opcode is commonly used to
 7362 //               indicate the type of machine instruction, while secondary
 7363 //               and tertiary are often used for prefix options or addressing
 7364 //               modes.
 7365 // ins_encode -- A list of encode classes with parameters. The encode class
 7366 //               name must have been defined in an 'enc_class' specification
 7367 //               in the encode section of the architecture description.
 7368 
 7369 // ============================================================================
 7370 // Memory (Load/Store) Instructions
 7371 
 7372 // Load Instructions
 7373 
 7374 // Load Byte (8 bit signed)
 7375 instruct loadB(iRegINoSp dst, memory1 mem)
 7376 %{
 7377   match(Set dst (LoadB mem));
 7378   predicate(!needs_acquiring_load(n));
 7379 
 7380   ins_cost(4 * INSN_COST);
 7381   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7382 
 7383   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7384 
 7385   ins_pipe(iload_reg_mem);
 7386 %}
 7387 
 7388 // Load Byte (8 bit signed) into long
 7389 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7390 %{
 7391   match(Set dst (ConvI2L (LoadB mem)));
 7392   predicate(!needs_acquiring_load(n->in(1)));
 7393 
 7394   ins_cost(4 * INSN_COST);
 7395   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7396 
 7397   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7398 
 7399   ins_pipe(iload_reg_mem);
 7400 %}
 7401 
 7402 // Load Byte (8 bit unsigned)
 7403 instruct loadUB(iRegINoSp dst, memory1 mem)
 7404 %{
 7405   match(Set dst (LoadUB mem));
 7406   predicate(!needs_acquiring_load(n));
 7407 
 7408   ins_cost(4 * INSN_COST);
 7409   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7410 
 7411   ins_encode(aarch64_enc_ldrb(dst, mem));
 7412 
 7413   ins_pipe(iload_reg_mem);
 7414 %}
 7415 
 7416 // Load Byte (8 bit unsigned) into long
 7417 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7418 %{
 7419   match(Set dst (ConvI2L (LoadUB mem)));
 7420   predicate(!needs_acquiring_load(n->in(1)));
 7421 
 7422   ins_cost(4 * INSN_COST);
 7423   format %{ "ldrb  $dst, $mem\t# byte" %}
 7424 
 7425   ins_encode(aarch64_enc_ldrb(dst, mem));
 7426 
 7427   ins_pipe(iload_reg_mem);
 7428 %}
 7429 
 7430 // Load Short (16 bit signed)
 7431 instruct loadS(iRegINoSp dst, memory2 mem)
 7432 %{
 7433   match(Set dst (LoadS mem));
 7434   predicate(!needs_acquiring_load(n));
 7435 
 7436   ins_cost(4 * INSN_COST);
 7437   format %{ "ldrshw  $dst, $mem\t# short" %}
 7438 
 7439   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7440 
 7441   ins_pipe(iload_reg_mem);
 7442 %}
 7443 
 7444 // Load Short (16 bit signed) into long
 7445 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7446 %{
 7447   match(Set dst (ConvI2L (LoadS mem)));
 7448   predicate(!needs_acquiring_load(n->in(1)));
 7449 
 7450   ins_cost(4 * INSN_COST);
 7451   format %{ "ldrsh  $dst, $mem\t# short" %}
 7452 
 7453   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7454 
 7455   ins_pipe(iload_reg_mem);
 7456 %}
 7457 
 7458 // Load Char (16 bit unsigned)
 7459 instruct loadUS(iRegINoSp dst, memory2 mem)
 7460 %{
 7461   match(Set dst (LoadUS mem));
 7462   predicate(!needs_acquiring_load(n));
 7463 
 7464   ins_cost(4 * INSN_COST);
 7465   format %{ "ldrh  $dst, $mem\t# short" %}
 7466 
 7467   ins_encode(aarch64_enc_ldrh(dst, mem));
 7468 
 7469   ins_pipe(iload_reg_mem);
 7470 %}
 7471 
 7472 // Load Short/Char (16 bit unsigned) into long
 7473 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7474 %{
 7475   match(Set dst (ConvI2L (LoadUS mem)));
 7476   predicate(!needs_acquiring_load(n->in(1)));
 7477 
 7478   ins_cost(4 * INSN_COST);
 7479   format %{ "ldrh  $dst, $mem\t# short" %}
 7480 
 7481   ins_encode(aarch64_enc_ldrh(dst, mem));
 7482 
 7483   ins_pipe(iload_reg_mem);
 7484 %}
 7485 
 7486 // Load Integer (32 bit signed)
 7487 instruct loadI(iRegINoSp dst, memory4 mem)
 7488 %{
 7489   match(Set dst (LoadI mem));
 7490   predicate(!needs_acquiring_load(n));
 7491 
 7492   ins_cost(4 * INSN_COST);
 7493   format %{ "ldrw  $dst, $mem\t# int" %}
 7494 
 7495   ins_encode(aarch64_enc_ldrw(dst, mem));
 7496 
 7497   ins_pipe(iload_reg_mem);
 7498 %}
 7499 
 7500 // Load Integer (32 bit signed) into long
 7501 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7502 %{
 7503   match(Set dst (ConvI2L (LoadI mem)));
 7504   predicate(!needs_acquiring_load(n->in(1)));
 7505 
 7506   ins_cost(4 * INSN_COST);
 7507   format %{ "ldrsw  $dst, $mem\t# int" %}
 7508 
 7509   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7510 
 7511   ins_pipe(iload_reg_mem);
 7512 %}
 7513 
 7514 // Load Integer (32 bit unsigned) into long
 7515 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7516 %{
 7517   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7518   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7519 
 7520   ins_cost(4 * INSN_COST);
 7521   format %{ "ldrw  $dst, $mem\t# int" %}
 7522 
 7523   ins_encode(aarch64_enc_ldrw(dst, mem));
 7524 
 7525   ins_pipe(iload_reg_mem);
 7526 %}
 7527 
 7528 // Load Long (64 bit signed)
 7529 instruct loadL(iRegLNoSp dst, memory8 mem)
 7530 %{
 7531   match(Set dst (LoadL mem));
 7532   predicate(!needs_acquiring_load(n));
 7533 
 7534   ins_cost(4 * INSN_COST);
 7535   format %{ "ldr  $dst, $mem\t# int" %}
 7536 
 7537   ins_encode(aarch64_enc_ldr(dst, mem));
 7538 
 7539   ins_pipe(iload_reg_mem);
 7540 %}
 7541 
 7542 // Load Range
 7543 instruct loadRange(iRegINoSp dst, memory4 mem)
 7544 %{
 7545   match(Set dst (LoadRange mem));
 7546 
 7547   ins_cost(4 * INSN_COST);
 7548   format %{ "ldrw  $dst, $mem\t# range" %}
 7549 
 7550   ins_encode(aarch64_enc_ldrw(dst, mem));
 7551 
 7552   ins_pipe(iload_reg_mem);
 7553 %}
 7554 
 7555 // Load Pointer
 7556 instruct loadP(iRegPNoSp dst, memory8 mem)
 7557 %{
 7558   match(Set dst (LoadP mem));
 7559   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7560 
 7561   ins_cost(4 * INSN_COST);
 7562   format %{ "ldr  $dst, $mem\t# ptr" %}
 7563 
 7564   ins_encode(aarch64_enc_ldr(dst, mem));
 7565 
 7566   ins_pipe(iload_reg_mem);
 7567 %}
 7568 
 7569 // Load Compressed Pointer
 7570 instruct loadN(iRegNNoSp dst, memory4 mem)
 7571 %{
 7572   match(Set dst (LoadN mem));
 7573   predicate(!needs_acquiring_load(n));
 7574 
 7575   ins_cost(4 * INSN_COST);
 7576   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7577 
 7578   ins_encode(aarch64_enc_ldrw(dst, mem));
 7579 
 7580   ins_pipe(iload_reg_mem);
 7581 %}
 7582 
 7583 // Load Klass Pointer
 7584 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7585 %{
 7586   match(Set dst (LoadKlass mem));
 7587   predicate(!needs_acquiring_load(n));
 7588 
 7589   ins_cost(4 * INSN_COST);
 7590   format %{ "ldr  $dst, $mem\t# class" %}
 7591 
 7592   ins_encode(aarch64_enc_ldr(dst, mem));
 7593 
 7594   ins_pipe(iload_reg_mem);
 7595 %}
 7596 
 7597 // Load Narrow Klass Pointer
 7598 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7599 %{
 7600   match(Set dst (LoadNKlass mem));
 7601   predicate(!needs_acquiring_load(n));
 7602 
 7603   ins_cost(4 * INSN_COST);
 7604   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7605 
 7606   ins_encode(aarch64_enc_ldrw(dst, mem));
 7607 
 7608   ins_pipe(iload_reg_mem);
 7609 %}
 7610 
 7611 // Load Float
 7612 instruct loadF(vRegF dst, memory4 mem)
 7613 %{
 7614   match(Set dst (LoadF mem));
 7615   predicate(!needs_acquiring_load(n));
 7616 
 7617   ins_cost(4 * INSN_COST);
 7618   format %{ "ldrs  $dst, $mem\t# float" %}
 7619 
 7620   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7621 
 7622   ins_pipe(pipe_class_memory);
 7623 %}
 7624 
 7625 // Load Double
 7626 instruct loadD(vRegD dst, memory8 mem)
 7627 %{
 7628   match(Set dst (LoadD mem));
 7629   predicate(!needs_acquiring_load(n));
 7630 
 7631   ins_cost(4 * INSN_COST);
 7632   format %{ "ldrd  $dst, $mem\t# double" %}
 7633 
 7634   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7635 
 7636   ins_pipe(pipe_class_memory);
 7637 %}
 7638 
 7639 
 7640 // Load Int Constant
 7641 instruct loadConI(iRegINoSp dst, immI src)
 7642 %{
 7643   match(Set dst src);
 7644 
 7645   ins_cost(INSN_COST);
 7646   format %{ "mov $dst, $src\t# int" %}
 7647 
 7648   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7649 
 7650   ins_pipe(ialu_imm);
 7651 %}
 7652 
 7653 // Load Long Constant
 7654 instruct loadConL(iRegLNoSp dst, immL src)
 7655 %{
 7656   match(Set dst src);
 7657 
 7658   ins_cost(INSN_COST);
 7659   format %{ "mov $dst, $src\t# long" %}
 7660 
 7661   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7662 
 7663   ins_pipe(ialu_imm);
 7664 %}
 7665 
 7666 // Load Pointer Constant
 7667 
 7668 instruct loadConP(iRegPNoSp dst, immP con)
 7669 %{
 7670   match(Set dst con);
 7671 
 7672   ins_cost(INSN_COST * 4);
 7673   format %{
 7674     "mov  $dst, $con\t# ptr\n\t"
 7675   %}
 7676 
 7677   ins_encode(aarch64_enc_mov_p(dst, con));
 7678 
 7679   ins_pipe(ialu_imm);
 7680 %}
 7681 
 7682 // Load Null Pointer Constant
 7683 
 7684 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7685 %{
 7686   match(Set dst con);
 7687 
 7688   ins_cost(INSN_COST);
 7689   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7690 
 7691   ins_encode(aarch64_enc_mov_p0(dst, con));
 7692 
 7693   ins_pipe(ialu_imm);
 7694 %}
 7695 
 7696 // Load Pointer Constant One
 7697 
 7698 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7699 %{
 7700   match(Set dst con);
 7701 
 7702   ins_cost(INSN_COST);
 7703   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7704 
 7705   ins_encode(aarch64_enc_mov_p1(dst, con));
 7706 
 7707   ins_pipe(ialu_imm);
 7708 %}
 7709 
 7710 // Load Byte Map Base Constant
 7711 
 7712 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7713 %{
 7714   match(Set dst con);
 7715 
 7716   ins_cost(INSN_COST);
 7717   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7718 
 7719   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7720 
 7721   ins_pipe(ialu_imm);
 7722 %}
 7723 
 7724 // Load Narrow Pointer Constant
 7725 
 7726 instruct loadConN(iRegNNoSp dst, immN con)
 7727 %{
 7728   match(Set dst con);
 7729 
 7730   ins_cost(INSN_COST * 4);
 7731   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7732 
 7733   ins_encode(aarch64_enc_mov_n(dst, con));
 7734 
 7735   ins_pipe(ialu_imm);
 7736 %}
 7737 
 7738 // Load Narrow Null Pointer Constant
 7739 
 7740 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7741 %{
 7742   match(Set dst con);
 7743 
 7744   ins_cost(INSN_COST);
 7745   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7746 
 7747   ins_encode(aarch64_enc_mov_n0(dst, con));
 7748 
 7749   ins_pipe(ialu_imm);
 7750 %}
 7751 
 7752 // Load Narrow Klass Constant
 7753 
 7754 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7755 %{
 7756   match(Set dst con);
 7757 
 7758   ins_cost(INSN_COST);
 7759   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7760 
 7761   ins_encode(aarch64_enc_mov_nk(dst, con));
 7762 
 7763   ins_pipe(ialu_imm);
 7764 %}
 7765 
 7766 // Load Packed Float Constant
 7767 
 7768 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7769   match(Set dst con);
 7770   ins_cost(INSN_COST * 4);
 7771   format %{ "fmovs  $dst, $con"%}
 7772   ins_encode %{
 7773     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7774   %}
 7775 
 7776   ins_pipe(fp_imm_s);
 7777 %}
 7778 
 7779 // Load Float Constant
 7780 
 7781 instruct loadConF(vRegF dst, immF con) %{
 7782   match(Set dst con);
 7783 
 7784   ins_cost(INSN_COST * 4);
 7785 
 7786   format %{
 7787     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7788   %}
 7789 
 7790   ins_encode %{
 7791     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7792   %}
 7793 
 7794   ins_pipe(fp_load_constant_s);
 7795 %}
 7796 
 7797 // Load Packed Double Constant
 7798 
 7799 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7800   match(Set dst con);
 7801   ins_cost(INSN_COST);
 7802   format %{ "fmovd  $dst, $con"%}
 7803   ins_encode %{
 7804     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7805   %}
 7806 
 7807   ins_pipe(fp_imm_d);
 7808 %}
 7809 
 7810 // Load Double Constant
 7811 
 7812 instruct loadConD(vRegD dst, immD con) %{
 7813   match(Set dst con);
 7814 
 7815   ins_cost(INSN_COST * 5);
 7816   format %{
 7817     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7818   %}
 7819 
 7820   ins_encode %{
 7821     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7822   %}
 7823 
 7824   ins_pipe(fp_load_constant_d);
 7825 %}
 7826 
 7827 // Store Instructions
 7828 
 7829 // Store CMS card-mark Immediate
 7830 instruct storeimmCM0(immI0 zero, memory1 mem)
 7831 %{
 7832   match(Set mem (StoreCM mem zero));
 7833 
 7834   ins_cost(INSN_COST);
 7835   format %{ "storestore (elided)\n\t"
 7836             "strb zr, $mem\t# byte" %}
 7837 
 7838   ins_encode(aarch64_enc_strb0(mem));
 7839 
 7840   ins_pipe(istore_mem);
 7841 %}
 7842 
 7843 // Store CMS card-mark Immediate with intervening StoreStore
 7844 // needed when using CMS with no conditional card marking
 7845 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7846 %{
 7847   match(Set mem (StoreCM mem zero));
 7848 
 7849   ins_cost(INSN_COST * 2);
 7850   format %{ "storestore\n\t"
 7851             "dmb ishst"
 7852             "\n\tstrb zr, $mem\t# byte" %}
 7853 
 7854   ins_encode(aarch64_enc_strb0_ordered(mem));
 7855 
 7856   ins_pipe(istore_mem);
 7857 %}
 7858 
 7859 // Store Byte
 7860 instruct storeB(iRegIorL2I src, memory1 mem)
 7861 %{
 7862   match(Set mem (StoreB mem src));
 7863   predicate(!needs_releasing_store(n));
 7864 
 7865   ins_cost(INSN_COST);
 7866   format %{ "strb  $src, $mem\t# byte" %}
 7867 
 7868   ins_encode(aarch64_enc_strb(src, mem));
 7869 
 7870   ins_pipe(istore_reg_mem);
 7871 %}
 7872 
 7873 
 7874 instruct storeimmB0(immI0 zero, memory1 mem)
 7875 %{
 7876   match(Set mem (StoreB mem zero));
 7877   predicate(!needs_releasing_store(n));
 7878 
 7879   ins_cost(INSN_COST);
 7880   format %{ "strb rscractch2, $mem\t# byte" %}
 7881 
 7882   ins_encode(aarch64_enc_strb0(mem));
 7883 
 7884   ins_pipe(istore_mem);
 7885 %}
 7886 
 7887 // Store Char/Short
 7888 instruct storeC(iRegIorL2I src, memory2 mem)
 7889 %{
 7890   match(Set mem (StoreC mem src));
 7891   predicate(!needs_releasing_store(n));
 7892 
 7893   ins_cost(INSN_COST);
 7894   format %{ "strh  $src, $mem\t# short" %}
 7895 
 7896   ins_encode(aarch64_enc_strh(src, mem));
 7897 
 7898   ins_pipe(istore_reg_mem);
 7899 %}
 7900 
 7901 instruct storeimmC0(immI0 zero, memory2 mem)
 7902 %{
 7903   match(Set mem (StoreC mem zero));
 7904   predicate(!needs_releasing_store(n));
 7905 
 7906   ins_cost(INSN_COST);
 7907   format %{ "strh  zr, $mem\t# short" %}
 7908 
 7909   ins_encode(aarch64_enc_strh0(mem));
 7910 
 7911   ins_pipe(istore_mem);
 7912 %}
 7913 
 7914 // Store Integer
 7915 
 7916 instruct storeI(iRegIorL2I src, memory4 mem)
 7917 %{
 7918   match(Set mem(StoreI mem src));
 7919   predicate(!needs_releasing_store(n));
 7920 
 7921   ins_cost(INSN_COST);
 7922   format %{ "strw  $src, $mem\t# int" %}
 7923 
 7924   ins_encode(aarch64_enc_strw(src, mem));
 7925 
 7926   ins_pipe(istore_reg_mem);
 7927 %}
 7928 
 7929 instruct storeimmI0(immI0 zero, memory4 mem)
 7930 %{
 7931   match(Set mem(StoreI mem zero));
 7932   predicate(!needs_releasing_store(n));
 7933 
 7934   ins_cost(INSN_COST);
 7935   format %{ "strw  zr, $mem\t# int" %}
 7936 
 7937   ins_encode(aarch64_enc_strw0(mem));
 7938 
 7939   ins_pipe(istore_mem);
 7940 %}
 7941 
 7942 // Store Long (64 bit signed)
 7943 instruct storeL(iRegL src, memory8 mem)
 7944 %{
 7945   match(Set mem (StoreL mem src));
 7946   predicate(!needs_releasing_store(n));
 7947 
 7948   ins_cost(INSN_COST);
 7949   format %{ "str  $src, $mem\t# int" %}
 7950 
 7951   ins_encode(aarch64_enc_str(src, mem));
 7952 
 7953   ins_pipe(istore_reg_mem);
 7954 %}
 7955 
 7956 // Store Long (64 bit signed)
 7957 instruct storeimmL0(immL0 zero, memory8 mem)
 7958 %{
 7959   match(Set mem (StoreL mem zero));
 7960   predicate(!needs_releasing_store(n));
 7961 
 7962   ins_cost(INSN_COST);
 7963   format %{ "str  zr, $mem\t# int" %}
 7964 
 7965   ins_encode(aarch64_enc_str0(mem));
 7966 
 7967   ins_pipe(istore_mem);
 7968 %}
 7969 
 7970 // Store Pointer
 7971 instruct storeP(iRegP src, memory8 mem)
 7972 %{
 7973   match(Set mem (StoreP mem src));
 7974   predicate(!needs_releasing_store(n));
 7975 
 7976   ins_cost(INSN_COST);
 7977   format %{ "str  $src, $mem\t# ptr" %}
 7978 
 7979   ins_encode(aarch64_enc_str(src, mem));
 7980 
 7981   ins_pipe(istore_reg_mem);
 7982 %}
 7983 
 7984 // Store Pointer
 7985 instruct storeimmP0(immP0 zero, memory8 mem)
 7986 %{
 7987   match(Set mem (StoreP mem zero));
 7988   predicate(!needs_releasing_store(n));
 7989 
 7990   ins_cost(INSN_COST);
 7991   format %{ "str zr, $mem\t# ptr" %}
 7992 
 7993   ins_encode(aarch64_enc_str0(mem));
 7994 
 7995   ins_pipe(istore_mem);
 7996 %}
 7997 
 7998 // Store Compressed Pointer
 7999 instruct storeN(iRegN src, memory4 mem)
 8000 %{
 8001   match(Set mem (StoreN mem src));
 8002   predicate(!needs_releasing_store(n));
 8003 
 8004   ins_cost(INSN_COST);
 8005   format %{ "strw  $src, $mem\t# compressed ptr" %}
 8006 
 8007   ins_encode(aarch64_enc_strw(src, mem));
 8008 
 8009   ins_pipe(istore_reg_mem);
 8010 %}
 8011 
 8012 instruct storeImmN0(immN0 zero, memory4 mem)
 8013 %{
 8014   match(Set mem (StoreN mem zero));
 8015   predicate(!needs_releasing_store(n));
 8016 
 8017   ins_cost(INSN_COST);
 8018   format %{ "strw  zr, $mem\t# compressed ptr" %}
 8019 
 8020   ins_encode(aarch64_enc_strw0(mem));
 8021 
 8022   ins_pipe(istore_mem);
 8023 %}
 8024 
 8025 // Store Float
 8026 instruct storeF(vRegF src, memory4 mem)
 8027 %{
 8028   match(Set mem (StoreF mem src));
 8029   predicate(!needs_releasing_store(n));
 8030 
 8031   ins_cost(INSN_COST);
 8032   format %{ "strs  $src, $mem\t# float" %}
 8033 
 8034   ins_encode( aarch64_enc_strs(src, mem) );
 8035 
 8036   ins_pipe(pipe_class_memory);
 8037 %}
 8038 
 8039 // TODO
 8040 // implement storeImmF0 and storeFImmPacked
 8041 
 8042 // Store Double
 8043 instruct storeD(vRegD src, memory8 mem)
 8044 %{
 8045   match(Set mem (StoreD mem src));
 8046   predicate(!needs_releasing_store(n));
 8047 
 8048   ins_cost(INSN_COST);
 8049   format %{ "strd  $src, $mem\t# double" %}
 8050 
 8051   ins_encode( aarch64_enc_strd(src, mem) );
 8052 
 8053   ins_pipe(pipe_class_memory);
 8054 %}
 8055 
 8056 // Store Compressed Klass Pointer
 8057 instruct storeNKlass(iRegN src, memory4 mem)
 8058 %{
 8059   predicate(!needs_releasing_store(n));
 8060   match(Set mem (StoreNKlass mem src));
 8061 
 8062   ins_cost(INSN_COST);
 8063   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 8064 
 8065   ins_encode(aarch64_enc_strw(src, mem));
 8066 
 8067   ins_pipe(istore_reg_mem);
 8068 %}
 8069 
 8070 // TODO
 8071 // implement storeImmD0 and storeDImmPacked
 8072 
 8073 // prefetch instructions
 8074 // Must be safe to execute with invalid address (cannot fault).
 8075 
 8076 instruct prefetchalloc( memory8 mem ) %{
 8077   match(PrefetchAllocation mem);
 8078 
 8079   ins_cost(INSN_COST);
 8080   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 8081 
 8082   ins_encode( aarch64_enc_prefetchw(mem) );
 8083 
 8084   ins_pipe(iload_prefetch);
 8085 %}
 8086 
 8087 //  ---------------- volatile loads and stores ----------------
 8088 
 8089 // Load Byte (8 bit signed)
 8090 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8091 %{
 8092   match(Set dst (LoadB mem));
 8093 
 8094   ins_cost(VOLATILE_REF_COST);
 8095   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8096 
 8097   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8098 
 8099   ins_pipe(pipe_serial);
 8100 %}
 8101 
 8102 // Load Byte (8 bit signed) into long
 8103 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8104 %{
 8105   match(Set dst (ConvI2L (LoadB mem)));
 8106 
 8107   ins_cost(VOLATILE_REF_COST);
 8108   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8109 
 8110   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8111 
 8112   ins_pipe(pipe_serial);
 8113 %}
 8114 
 8115 // Load Byte (8 bit unsigned)
 8116 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8117 %{
 8118   match(Set dst (LoadUB mem));
 8119 
 8120   ins_cost(VOLATILE_REF_COST);
 8121   format %{ "ldarb  $dst, $mem\t# byte" %}
 8122 
 8123   ins_encode(aarch64_enc_ldarb(dst, mem));
 8124 
 8125   ins_pipe(pipe_serial);
 8126 %}
 8127 
 8128 // Load Byte (8 bit unsigned) into long
 8129 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8130 %{
 8131   match(Set dst (ConvI2L (LoadUB mem)));
 8132 
 8133   ins_cost(VOLATILE_REF_COST);
 8134   format %{ "ldarb  $dst, $mem\t# byte" %}
 8135 
 8136   ins_encode(aarch64_enc_ldarb(dst, mem));
 8137 
 8138   ins_pipe(pipe_serial);
 8139 %}
 8140 
 8141 // Load Short (16 bit signed)
 8142 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8143 %{
 8144   match(Set dst (LoadS mem));
 8145 
 8146   ins_cost(VOLATILE_REF_COST);
 8147   format %{ "ldarshw  $dst, $mem\t# short" %}
 8148 
 8149   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8150 
 8151   ins_pipe(pipe_serial);
 8152 %}
 8153 
 8154 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8155 %{
 8156   match(Set dst (LoadUS mem));
 8157 
 8158   ins_cost(VOLATILE_REF_COST);
 8159   format %{ "ldarhw  $dst, $mem\t# short" %}
 8160 
 8161   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8162 
 8163   ins_pipe(pipe_serial);
 8164 %}
 8165 
 8166 // Load Short/Char (16 bit unsigned) into long
 8167 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8168 %{
 8169   match(Set dst (ConvI2L (LoadUS mem)));
 8170 
 8171   ins_cost(VOLATILE_REF_COST);
 8172   format %{ "ldarh  $dst, $mem\t# short" %}
 8173 
 8174   ins_encode(aarch64_enc_ldarh(dst, mem));
 8175 
 8176   ins_pipe(pipe_serial);
 8177 %}
 8178 
 8179 // Load Short/Char (16 bit signed) into long
 8180 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8181 %{
 8182   match(Set dst (ConvI2L (LoadS mem)));
 8183 
 8184   ins_cost(VOLATILE_REF_COST);
 8185   format %{ "ldarh  $dst, $mem\t# short" %}
 8186 
 8187   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8188 
 8189   ins_pipe(pipe_serial);
 8190 %}
 8191 
 8192 // Load Integer (32 bit signed)
 8193 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8194 %{
 8195   match(Set dst (LoadI mem));
 8196 
 8197   ins_cost(VOLATILE_REF_COST);
 8198   format %{ "ldarw  $dst, $mem\t# int" %}
 8199 
 8200   ins_encode(aarch64_enc_ldarw(dst, mem));
 8201 
 8202   ins_pipe(pipe_serial);
 8203 %}
 8204 
 8205 // Load Integer (32 bit unsigned) into long
 8206 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8207 %{
 8208   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8209 
 8210   ins_cost(VOLATILE_REF_COST);
 8211   format %{ "ldarw  $dst, $mem\t# int" %}
 8212 
 8213   ins_encode(aarch64_enc_ldarw(dst, mem));
 8214 
 8215   ins_pipe(pipe_serial);
 8216 %}
 8217 
 8218 // Load Long (64 bit signed)
 8219 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8220 %{
 8221   match(Set dst (LoadL mem));
 8222 
 8223   ins_cost(VOLATILE_REF_COST);
 8224   format %{ "ldar  $dst, $mem\t# int" %}
 8225 
 8226   ins_encode(aarch64_enc_ldar(dst, mem));
 8227 
 8228   ins_pipe(pipe_serial);
 8229 %}
 8230 
 8231 // Load Pointer
 8232 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8233 %{
 8234   match(Set dst (LoadP mem));
 8235   predicate(n->as_Load()->barrier_data() == 0);
 8236 
 8237   ins_cost(VOLATILE_REF_COST);
 8238   format %{ "ldar  $dst, $mem\t# ptr" %}
 8239 
 8240   ins_encode(aarch64_enc_ldar(dst, mem));
 8241 
 8242   ins_pipe(pipe_serial);
 8243 %}
 8244 
 8245 // Load Compressed Pointer
 8246 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8247 %{
 8248   match(Set dst (LoadN mem));
 8249 
 8250   ins_cost(VOLATILE_REF_COST);
 8251   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8252 
 8253   ins_encode(aarch64_enc_ldarw(dst, mem));
 8254 
 8255   ins_pipe(pipe_serial);
 8256 %}
 8257 
 8258 // Load Float
 8259 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8260 %{
 8261   match(Set dst (LoadF mem));
 8262 
 8263   ins_cost(VOLATILE_REF_COST);
 8264   format %{ "ldars  $dst, $mem\t# float" %}
 8265 
 8266   ins_encode( aarch64_enc_fldars(dst, mem) );
 8267 
 8268   ins_pipe(pipe_serial);
 8269 %}
 8270 
 8271 // Load Double
 8272 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8273 %{
 8274   match(Set dst (LoadD mem));
 8275 
 8276   ins_cost(VOLATILE_REF_COST);
 8277   format %{ "ldard  $dst, $mem\t# double" %}
 8278 
 8279   ins_encode( aarch64_enc_fldard(dst, mem) );
 8280 
 8281   ins_pipe(pipe_serial);
 8282 %}
 8283 
 8284 // Store Byte
 8285 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8286 %{
 8287   match(Set mem (StoreB mem src));
 8288 
 8289   ins_cost(VOLATILE_REF_COST);
 8290   format %{ "stlrb  $src, $mem\t# byte" %}
 8291 
 8292   ins_encode(aarch64_enc_stlrb(src, mem));
 8293 
 8294   ins_pipe(pipe_class_memory);
 8295 %}
 8296 
 8297 // Store Char/Short
 8298 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8299 %{
 8300   match(Set mem (StoreC mem src));
 8301 
 8302   ins_cost(VOLATILE_REF_COST);
 8303   format %{ "stlrh  $src, $mem\t# short" %}
 8304 
 8305   ins_encode(aarch64_enc_stlrh(src, mem));
 8306 
 8307   ins_pipe(pipe_class_memory);
 8308 %}
 8309 
 8310 // Store Integer
 8311 
 8312 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8313 %{
 8314   match(Set mem(StoreI mem src));
 8315 
 8316   ins_cost(VOLATILE_REF_COST);
 8317   format %{ "stlrw  $src, $mem\t# int" %}
 8318 
 8319   ins_encode(aarch64_enc_stlrw(src, mem));
 8320 
 8321   ins_pipe(pipe_class_memory);
 8322 %}
 8323 
 8324 // Store Long (64 bit signed)
 8325 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8326 %{
 8327   match(Set mem (StoreL mem src));
 8328 
 8329   ins_cost(VOLATILE_REF_COST);
 8330   format %{ "stlr  $src, $mem\t# int" %}
 8331 
 8332   ins_encode(aarch64_enc_stlr(src, mem));
 8333 
 8334   ins_pipe(pipe_class_memory);
 8335 %}
 8336 
 8337 // Store Pointer
 8338 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8339 %{
 8340   match(Set mem (StoreP mem src));
 8341 
 8342   ins_cost(VOLATILE_REF_COST);
 8343   format %{ "stlr  $src, $mem\t# ptr" %}
 8344 
 8345   ins_encode(aarch64_enc_stlr(src, mem));
 8346 
 8347   ins_pipe(pipe_class_memory);
 8348 %}
 8349 
 8350 // Store Compressed Pointer
 8351 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8352 %{
 8353   match(Set mem (StoreN mem src));
 8354 
 8355   ins_cost(VOLATILE_REF_COST);
 8356   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8357 
 8358   ins_encode(aarch64_enc_stlrw(src, mem));
 8359 
 8360   ins_pipe(pipe_class_memory);
 8361 %}
 8362 
 8363 // Store Float
 8364 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8365 %{
 8366   match(Set mem (StoreF mem src));
 8367 
 8368   ins_cost(VOLATILE_REF_COST);
 8369   format %{ "stlrs  $src, $mem\t# float" %}
 8370 
 8371   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8372 
 8373   ins_pipe(pipe_class_memory);
 8374 %}
 8375 
 8376 // TODO
 8377 // implement storeImmF0 and storeFImmPacked
 8378 
 8379 // Store Double
 8380 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8381 %{
 8382   match(Set mem (StoreD mem src));
 8383 
 8384   ins_cost(VOLATILE_REF_COST);
 8385   format %{ "stlrd  $src, $mem\t# double" %}
 8386 
 8387   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8388 
 8389   ins_pipe(pipe_class_memory);
 8390 %}
 8391 
 8392 //  ---------------- end of volatile loads and stores ----------------
 8393 
 8394 instruct cacheWB(indirect addr)
 8395 %{
 8396   predicate(VM_Version::supports_data_cache_line_flush());
 8397   match(CacheWB addr);
 8398 
 8399   ins_cost(100);
 8400   format %{"cache wb $addr" %}
 8401   ins_encode %{
 8402     assert($addr->index_position() < 0, "should be");
 8403     assert($addr$$disp == 0, "should be");
 8404     __ cache_wb(Address($addr$$base$$Register, 0));
 8405   %}
 8406   ins_pipe(pipe_slow); // XXX
 8407 %}
 8408 
 8409 instruct cacheWBPreSync()
 8410 %{
 8411   predicate(VM_Version::supports_data_cache_line_flush());
 8412   match(CacheWBPreSync);
 8413 
 8414   ins_cost(100);
 8415   format %{"cache wb presync" %}
 8416   ins_encode %{
 8417     __ cache_wbsync(true);
 8418   %}
 8419   ins_pipe(pipe_slow); // XXX
 8420 %}
 8421 
 8422 instruct cacheWBPostSync()
 8423 %{
 8424   predicate(VM_Version::supports_data_cache_line_flush());
 8425   match(CacheWBPostSync);
 8426 
 8427   ins_cost(100);
 8428   format %{"cache wb postsync" %}
 8429   ins_encode %{
 8430     __ cache_wbsync(false);
 8431   %}
 8432   ins_pipe(pipe_slow); // XXX
 8433 %}
 8434 
 8435 // ============================================================================
 8436 // BSWAP Instructions
 8437 
 8438 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8439   match(Set dst (ReverseBytesI src));
 8440 
 8441   ins_cost(INSN_COST);
 8442   format %{ "revw  $dst, $src" %}
 8443 
 8444   ins_encode %{
 8445     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8446   %}
 8447 
 8448   ins_pipe(ialu_reg);
 8449 %}
 8450 
 8451 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8452   match(Set dst (ReverseBytesL src));
 8453 
 8454   ins_cost(INSN_COST);
 8455   format %{ "rev  $dst, $src" %}
 8456 
 8457   ins_encode %{
 8458     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8459   %}
 8460 
 8461   ins_pipe(ialu_reg);
 8462 %}
 8463 
 8464 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8465   match(Set dst (ReverseBytesUS src));
 8466 
 8467   ins_cost(INSN_COST);
 8468   format %{ "rev16w  $dst, $src" %}
 8469 
 8470   ins_encode %{
 8471     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8472   %}
 8473 
 8474   ins_pipe(ialu_reg);
 8475 %}
 8476 
 8477 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8478   match(Set dst (ReverseBytesS src));
 8479 
 8480   ins_cost(INSN_COST);
 8481   format %{ "rev16w  $dst, $src\n\t"
 8482             "sbfmw $dst, $dst, #0, #15" %}
 8483 
 8484   ins_encode %{
 8485     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8486     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8487   %}
 8488 
 8489   ins_pipe(ialu_reg);
 8490 %}
 8491 
 8492 // ============================================================================
 8493 // Zero Count Instructions
 8494 
 8495 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8496   match(Set dst (CountLeadingZerosI src));
 8497 
 8498   ins_cost(INSN_COST);
 8499   format %{ "clzw  $dst, $src" %}
 8500   ins_encode %{
 8501     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8502   %}
 8503 
 8504   ins_pipe(ialu_reg);
 8505 %}
 8506 
 8507 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8508   match(Set dst (CountLeadingZerosL src));
 8509 
 8510   ins_cost(INSN_COST);
 8511   format %{ "clz   $dst, $src" %}
 8512   ins_encode %{
 8513     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8514   %}
 8515 
 8516   ins_pipe(ialu_reg);
 8517 %}
 8518 
 8519 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8520   match(Set dst (CountTrailingZerosI src));
 8521 
 8522   ins_cost(INSN_COST * 2);
 8523   format %{ "rbitw  $dst, $src\n\t"
 8524             "clzw   $dst, $dst" %}
 8525   ins_encode %{
 8526     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8527     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8528   %}
 8529 
 8530   ins_pipe(ialu_reg);
 8531 %}
 8532 
 8533 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8534   match(Set dst (CountTrailingZerosL src));
 8535 
 8536   ins_cost(INSN_COST * 2);
 8537   format %{ "rbit   $dst, $src\n\t"
 8538             "clz    $dst, $dst" %}
 8539   ins_encode %{
 8540     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8541     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8542   %}
 8543 
 8544   ins_pipe(ialu_reg);
 8545 %}
 8546 
 8547 //---------- Population Count Instructions -------------------------------------
 8548 //
 8549 
 8550 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8551   predicate(UsePopCountInstruction);
 8552   match(Set dst (PopCountI src));
 8553   effect(TEMP tmp);
 8554   ins_cost(INSN_COST * 13);
 8555 
 8556   format %{ "movw   $src, $src\n\t"
 8557             "mov    $tmp, $src\t# vector (1D)\n\t"
 8558             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8559             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8560             "mov    $dst, $tmp\t# vector (1D)" %}
 8561   ins_encode %{
 8562     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8563     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8564     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8565     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8566     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8567   %}
 8568 
 8569   ins_pipe(pipe_class_default);
 8570 %}
 8571 
 8572 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8573   predicate(UsePopCountInstruction);
 8574   match(Set dst (PopCountI (LoadI mem)));
 8575   effect(TEMP tmp);
 8576   ins_cost(INSN_COST * 13);
 8577 
 8578   format %{ "ldrs   $tmp, $mem\n\t"
 8579             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8580             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8581             "mov    $dst, $tmp\t# vector (1D)" %}
 8582   ins_encode %{
 8583     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8584     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8585               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8586     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8587     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8588     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8589   %}
 8590 
 8591   ins_pipe(pipe_class_default);
 8592 %}
 8593 
 8594 // Note: Long.bitCount(long) returns an int.
 8595 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8596   predicate(UsePopCountInstruction);
 8597   match(Set dst (PopCountL src));
 8598   effect(TEMP tmp);
 8599   ins_cost(INSN_COST * 13);
 8600 
 8601   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8602             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8603             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8604             "mov    $dst, $tmp\t# vector (1D)" %}
 8605   ins_encode %{
 8606     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8607     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8608     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8609     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8610   %}
 8611 
 8612   ins_pipe(pipe_class_default);
 8613 %}
 8614 
 8615 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8616   predicate(UsePopCountInstruction);
 8617   match(Set dst (PopCountL (LoadL mem)));
 8618   effect(TEMP tmp);
 8619   ins_cost(INSN_COST * 13);
 8620 
 8621   format %{ "ldrd   $tmp, $mem\n\t"
 8622             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8623             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8624             "mov    $dst, $tmp\t# vector (1D)" %}
 8625   ins_encode %{
 8626     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8627     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8628               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8629     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8630     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8631     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8632   %}
 8633 
 8634   ins_pipe(pipe_class_default);
 8635 %}
 8636 
 8637 // ============================================================================
 8638 // MemBar Instruction
 8639 
 8640 instruct load_fence() %{
 8641   match(LoadFence);
 8642   ins_cost(VOLATILE_REF_COST);
 8643 
 8644   format %{ "load_fence" %}
 8645 
 8646   ins_encode %{
 8647     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8648   %}
 8649   ins_pipe(pipe_serial);
 8650 %}
 8651 
 8652 instruct unnecessary_membar_acquire() %{
 8653   predicate(unnecessary_acquire(n));
 8654   match(MemBarAcquire);
 8655   ins_cost(0);
 8656 
 8657   format %{ "membar_acquire (elided)" %}
 8658 
 8659   ins_encode %{
 8660     __ block_comment("membar_acquire (elided)");
 8661   %}
 8662 
 8663   ins_pipe(pipe_class_empty);
 8664 %}
 8665 
 8666 instruct membar_acquire() %{
 8667   match(MemBarAcquire);
 8668   ins_cost(VOLATILE_REF_COST);
 8669 
 8670   format %{ "membar_acquire\n\t"
 8671             "dmb ish" %}
 8672 
 8673   ins_encode %{
 8674     __ block_comment("membar_acquire");
 8675     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8676   %}
 8677 
 8678   ins_pipe(pipe_serial);
 8679 %}
 8680 
 8681 
 8682 instruct membar_acquire_lock() %{
 8683   match(MemBarAcquireLock);
 8684   ins_cost(VOLATILE_REF_COST);
 8685 
 8686   format %{ "membar_acquire_lock (elided)" %}
 8687 
 8688   ins_encode %{
 8689     __ block_comment("membar_acquire_lock (elided)");
 8690   %}
 8691 
 8692   ins_pipe(pipe_serial);
 8693 %}
 8694 
 8695 instruct store_fence() %{
 8696   match(StoreFence);
 8697   ins_cost(VOLATILE_REF_COST);
 8698 
 8699   format %{ "store_fence" %}
 8700 
 8701   ins_encode %{
 8702     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8703   %}
 8704   ins_pipe(pipe_serial);
 8705 %}
 8706 
 8707 instruct unnecessary_membar_release() %{
 8708   predicate(unnecessary_release(n));
 8709   match(MemBarRelease);
 8710   ins_cost(0);
 8711 
 8712   format %{ "membar_release (elided)" %}
 8713 
 8714   ins_encode %{
 8715     __ block_comment("membar_release (elided)");
 8716   %}
 8717   ins_pipe(pipe_serial);
 8718 %}
 8719 
 8720 instruct membar_release() %{
 8721   match(MemBarRelease);
 8722   ins_cost(VOLATILE_REF_COST);
 8723 
 8724   format %{ "membar_release\n\t"
 8725             "dmb ish" %}
 8726 
 8727   ins_encode %{
 8728     __ block_comment("membar_release");
 8729     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8730   %}
 8731   ins_pipe(pipe_serial);
 8732 %}
 8733 
 8734 instruct membar_storestore() %{
 8735   match(MemBarStoreStore);
 8736   match(StoreStoreFence);
 8737   ins_cost(VOLATILE_REF_COST);
 8738 
 8739   format %{ "MEMBAR-store-store" %}
 8740 
 8741   ins_encode %{
 8742     __ membar(Assembler::StoreStore);
 8743   %}
 8744   ins_pipe(pipe_serial);
 8745 %}
 8746 
 8747 instruct membar_release_lock() %{
 8748   match(MemBarReleaseLock);
 8749   ins_cost(VOLATILE_REF_COST);
 8750 
 8751   format %{ "membar_release_lock (elided)" %}
 8752 
 8753   ins_encode %{
 8754     __ block_comment("membar_release_lock (elided)");
 8755   %}
 8756 
 8757   ins_pipe(pipe_serial);
 8758 %}
 8759 
 8760 instruct unnecessary_membar_volatile() %{
 8761   predicate(unnecessary_volatile(n));
 8762   match(MemBarVolatile);
 8763   ins_cost(0);
 8764 
 8765   format %{ "membar_volatile (elided)" %}
 8766 
 8767   ins_encode %{
 8768     __ block_comment("membar_volatile (elided)");
 8769   %}
 8770 
 8771   ins_pipe(pipe_serial);
 8772 %}
 8773 
 8774 instruct membar_volatile() %{
 8775   match(MemBarVolatile);
 8776   ins_cost(VOLATILE_REF_COST*100);
 8777 
 8778   format %{ "membar_volatile\n\t"
 8779              "dmb ish"%}
 8780 
 8781   ins_encode %{
 8782     __ block_comment("membar_volatile");
 8783     __ membar(Assembler::StoreLoad);
 8784   %}
 8785 
 8786   ins_pipe(pipe_serial);
 8787 %}
 8788 
 8789 // ============================================================================
 8790 // Cast/Convert Instructions
 8791 
 8792 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8793   match(Set dst (CastX2P src));
 8794 
 8795   ins_cost(INSN_COST);
 8796   format %{ "mov $dst, $src\t# long -> ptr" %}
 8797 
 8798   ins_encode %{
 8799     if ($dst$$reg != $src$$reg) {
 8800       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8801     }
 8802   %}
 8803 
 8804   ins_pipe(ialu_reg);
 8805 %}
 8806 
 8807 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8808   match(Set dst (CastP2X src));
 8809 
 8810   ins_cost(INSN_COST);
 8811   format %{ "mov $dst, $src\t# ptr -> long" %}
 8812 
 8813   ins_encode %{
 8814     if ($dst$$reg != $src$$reg) {
 8815       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8816     }
 8817   %}
 8818 
 8819   ins_pipe(ialu_reg);
 8820 %}
 8821 
 8822 // Convert oop into int for vectors alignment masking
 8823 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8824   match(Set dst (ConvL2I (CastP2X src)));
 8825 
 8826   ins_cost(INSN_COST);
 8827   format %{ "movw $dst, $src\t# ptr -> int" %}
 8828   ins_encode %{
 8829     __ movw($dst$$Register, $src$$Register);
 8830   %}
 8831 
 8832   ins_pipe(ialu_reg);
 8833 %}
 8834 
 8835 // Convert compressed oop into int for vectors alignment masking
 8836 // in case of 32bit oops (heap < 4Gb).
 8837 instruct convN2I(iRegINoSp dst, iRegN src)
 8838 %{
 8839   predicate(CompressedOops::shift() == 0);
 8840   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8841 
 8842   ins_cost(INSN_COST);
 8843   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8844   ins_encode %{
 8845     __ movw($dst$$Register, $src$$Register);
 8846   %}
 8847 
 8848   ins_pipe(ialu_reg);
 8849 %}
 8850 
 8851 
 8852 // Convert oop pointer into compressed form
 8853 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8854   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8855   match(Set dst (EncodeP src));
 8856   effect(KILL cr);
 8857   ins_cost(INSN_COST * 3);
 8858   format %{ "encode_heap_oop $dst, $src" %}
 8859   ins_encode %{
 8860     Register s = $src$$Register;
 8861     Register d = $dst$$Register;
 8862     __ encode_heap_oop(d, s);
 8863   %}
 8864   ins_pipe(ialu_reg);
 8865 %}
 8866 
 8867 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8868   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8869   match(Set dst (EncodeP src));
 8870   ins_cost(INSN_COST * 3);
 8871   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8872   ins_encode %{
 8873     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8874   %}
 8875   ins_pipe(ialu_reg);
 8876 %}
 8877 
 8878 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8879   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8880             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8881   match(Set dst (DecodeN src));
 8882   ins_cost(INSN_COST * 3);
 8883   format %{ "decode_heap_oop $dst, $src" %}
 8884   ins_encode %{
 8885     Register s = $src$$Register;
 8886     Register d = $dst$$Register;
 8887     __ decode_heap_oop(d, s);
 8888   %}
 8889   ins_pipe(ialu_reg);
 8890 %}
 8891 
 8892 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8893   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8894             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8895   match(Set dst (DecodeN src));
 8896   ins_cost(INSN_COST * 3);
 8897   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8898   ins_encode %{
 8899     Register s = $src$$Register;
 8900     Register d = $dst$$Register;
 8901     __ decode_heap_oop_not_null(d, s);
 8902   %}
 8903   ins_pipe(ialu_reg);
 8904 %}
 8905 
 8906 // n.b. AArch64 implementations of encode_klass_not_null and
 8907 // decode_klass_not_null do not modify the flags register so, unlike
 8908 // Intel, we don't kill CR as a side effect here
 8909 
 8910 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8911   match(Set dst (EncodePKlass src));
 8912 
 8913   ins_cost(INSN_COST * 3);
 8914   format %{ "encode_klass_not_null $dst,$src" %}
 8915 
 8916   ins_encode %{
 8917     Register src_reg = as_Register($src$$reg);
 8918     Register dst_reg = as_Register($dst$$reg);
 8919     __ encode_klass_not_null(dst_reg, src_reg);
 8920   %}
 8921 
 8922    ins_pipe(ialu_reg);
 8923 %}
 8924 
 8925 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8926   match(Set dst (DecodeNKlass src));
 8927 
 8928   ins_cost(INSN_COST * 3);
 8929   format %{ "decode_klass_not_null $dst,$src" %}
 8930 
 8931   ins_encode %{
 8932     Register src_reg = as_Register($src$$reg);
 8933     Register dst_reg = as_Register($dst$$reg);
 8934     if (dst_reg != src_reg) {
 8935       __ decode_klass_not_null(dst_reg, src_reg);
 8936     } else {
 8937       __ decode_klass_not_null(dst_reg);
 8938     }
 8939   %}
 8940 
 8941    ins_pipe(ialu_reg);
 8942 %}
 8943 
 8944 instruct checkCastPP(iRegPNoSp dst)
 8945 %{
 8946   match(Set dst (CheckCastPP dst));
 8947 
 8948   size(0);
 8949   format %{ "# checkcastPP of $dst" %}
 8950   ins_encode(/* empty encoding */);
 8951   ins_pipe(pipe_class_empty);
 8952 %}
 8953 
 8954 instruct castPP(iRegPNoSp dst)
 8955 %{
 8956   match(Set dst (CastPP dst));
 8957 
 8958   size(0);
 8959   format %{ "# castPP of $dst" %}
 8960   ins_encode(/* empty encoding */);
 8961   ins_pipe(pipe_class_empty);
 8962 %}
 8963 
 8964 instruct castII(iRegI dst)
 8965 %{
 8966   match(Set dst (CastII dst));
 8967 
 8968   size(0);
 8969   format %{ "# castII of $dst" %}
 8970   ins_encode(/* empty encoding */);
 8971   ins_cost(0);
 8972   ins_pipe(pipe_class_empty);
 8973 %}
 8974 
 8975 instruct castLL(iRegL dst)
 8976 %{
 8977   match(Set dst (CastLL dst));
 8978 
 8979   size(0);
 8980   format %{ "# castLL of $dst" %}
 8981   ins_encode(/* empty encoding */);
 8982   ins_cost(0);
 8983   ins_pipe(pipe_class_empty);
 8984 %}
 8985 
 8986 instruct castFF(vRegF dst)
 8987 %{
 8988   match(Set dst (CastFF dst));
 8989 
 8990   size(0);
 8991   format %{ "# castFF of $dst" %}
 8992   ins_encode(/* empty encoding */);
 8993   ins_cost(0);
 8994   ins_pipe(pipe_class_empty);
 8995 %}
 8996 
 8997 instruct castDD(vRegD dst)
 8998 %{
 8999   match(Set dst (CastDD dst));
 9000 
 9001   size(0);
 9002   format %{ "# castDD of $dst" %}
 9003   ins_encode(/* empty encoding */);
 9004   ins_cost(0);
 9005   ins_pipe(pipe_class_empty);
 9006 %}
 9007 
 9008 instruct castVVD(vecD dst)
 9009 %{
 9010   match(Set dst (CastVV dst));
 9011 
 9012   size(0);
 9013   format %{ "# castVV of $dst" %}
 9014   ins_encode(/* empty encoding */);
 9015   ins_cost(0);
 9016   ins_pipe(pipe_class_empty);
 9017 %}
 9018 
 9019 instruct castVVX(vecX dst)
 9020 %{
 9021   match(Set dst (CastVV dst));
 9022 
 9023   size(0);
 9024   format %{ "# castVV of $dst" %}
 9025   ins_encode(/* empty encoding */);
 9026   ins_cost(0);
 9027   ins_pipe(pipe_class_empty);
 9028 %}
 9029 
 9030 instruct castVV(vReg dst)
 9031 %{
 9032   match(Set dst (CastVV dst));
 9033 
 9034   size(0);
 9035   format %{ "# castVV of $dst" %}
 9036   ins_encode(/* empty encoding */);
 9037   ins_cost(0);
 9038   ins_pipe(pipe_class_empty);
 9039 %}
 9040 
 9041 instruct castVVMask(pRegGov dst)
 9042 %{
 9043   match(Set dst (CastVV dst));
 9044 
 9045   size(0);
 9046   format %{ "# castVV of $dst" %}
 9047   ins_encode(/* empty encoding */);
 9048   ins_cost(0);
 9049   ins_pipe(pipe_class_empty);
 9050 %}
 9051 
 9052 // ============================================================================
 9053 // Atomic operation instructions
 9054 //
 9055 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9056 // Store{PIL}Conditional instructions using a normal load for the
 9057 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9058 //
 9059 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9060 // pair to lock object allocations from Eden space when not using
 9061 // TLABs.
 9062 //
 9063 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9064 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9065 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9066 // only for 64-bit.
 9067 //
 9068 // We implement LoadPLocked and StorePLocked instructions using,
 9069 // respectively the AArch64 hw load-exclusive and store-conditional
 9070 // instructions. Whereas we must implement each of
 9071 // Store{IL}Conditional using a CAS which employs a pair of
 9072 // instructions comprising a load-exclusive followed by a
 9073 // store-conditional.
 9074 
 9075 
 9076 // Locked-load (linked load) of the current heap-top
 9077 // used when updating the eden heap top
 9078 // implemented using ldaxr on AArch64
 9079 
 9080 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9081 %{
 9082   match(Set dst (LoadPLocked mem));
 9083 
 9084   ins_cost(VOLATILE_REF_COST);
 9085 
 9086   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9087 
 9088   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9089 
 9090   ins_pipe(pipe_serial);
 9091 %}
 9092 
 9093 // Conditional-store of the updated heap-top.
 9094 // Used during allocation of the shared heap.
 9095 // Sets flag (EQ) on success.
 9096 // implemented using stlxr on AArch64.
 9097 
 9098 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9099 %{
 9100   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9101 
 9102   ins_cost(VOLATILE_REF_COST);
 9103 
 9104  // TODO
 9105  // do we need to do a store-conditional release or can we just use a
 9106  // plain store-conditional?
 9107 
 9108   format %{
 9109     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9110     "cmpw rscratch1, zr\t# EQ on successful write"
 9111   %}
 9112 
 9113   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9114 
 9115   ins_pipe(pipe_serial);
 9116 %}
 9117 
 9118 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9119 %{
 9120   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9121 
 9122   ins_cost(VOLATILE_REF_COST);
 9123 
 9124   format %{
 9125     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9126     "cmpw rscratch1, zr\t# EQ on successful write"
 9127   %}
 9128 
 9129   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9130 
 9131   ins_pipe(pipe_slow);
 9132 %}
 9133 
 9134 // storeIConditional also has acquire semantics, for no better reason
 9135 // than matching storeLConditional.  At the time of writing this
 9136 // comment storeIConditional was not used anywhere by AArch64.
 9137 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9138 %{
 9139   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9140 
 9141   ins_cost(VOLATILE_REF_COST);
 9142 
 9143   format %{
 9144     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9145     "cmpw rscratch1, zr\t# EQ on successful write"
 9146   %}
 9147 
 9148   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9149 
 9150   ins_pipe(pipe_slow);
 9151 %}
 9152 
 9153 // standard CompareAndSwapX when we are using barriers
 9154 // these have higher priority than the rules selected by a predicate
 9155 
 9156 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9157 // can't match them
 9158 
 9159 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9160 
 9161   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9162   ins_cost(2 * VOLATILE_REF_COST);
 9163 
 9164   effect(KILL cr);
 9165 
 9166   format %{
 9167     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9168     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9169   %}
 9170 
 9171   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9172             aarch64_enc_cset_eq(res));
 9173 
 9174   ins_pipe(pipe_slow);
 9175 %}
 9176 
 9177 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9178 
 9179   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9180   ins_cost(2 * VOLATILE_REF_COST);
 9181 
 9182   effect(KILL cr);
 9183 
 9184   format %{
 9185     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9186     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9187   %}
 9188 
 9189   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9190             aarch64_enc_cset_eq(res));
 9191 
 9192   ins_pipe(pipe_slow);
 9193 %}
 9194 
 9195 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9196 
 9197   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9198   ins_cost(2 * VOLATILE_REF_COST);
 9199 
 9200   effect(KILL cr);
 9201 
 9202  format %{
 9203     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9204     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9205  %}
 9206 
 9207  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9208             aarch64_enc_cset_eq(res));
 9209 
 9210   ins_pipe(pipe_slow);
 9211 %}
 9212 
 9213 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9214 
 9215   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9216   ins_cost(2 * VOLATILE_REF_COST);
 9217 
 9218   effect(KILL cr);
 9219 
 9220  format %{
 9221     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9222     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9223  %}
 9224 
 9225  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9226             aarch64_enc_cset_eq(res));
 9227 
 9228   ins_pipe(pipe_slow);
 9229 %}
 9230 
 9231 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9232 
 9233   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9234   predicate(n->as_LoadStore()->barrier_data() == 0);
 9235   ins_cost(2 * VOLATILE_REF_COST);
 9236 
 9237   effect(KILL cr);
 9238 
 9239  format %{
 9240     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9241     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9242  %}
 9243 
 9244  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9245             aarch64_enc_cset_eq(res));
 9246 
 9247   ins_pipe(pipe_slow);
 9248 %}
 9249 
 9250 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9251 
 9252   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9253   ins_cost(2 * VOLATILE_REF_COST);
 9254 
 9255   effect(KILL cr);
 9256 
 9257  format %{
 9258     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9259     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9260  %}
 9261 
 9262  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9263             aarch64_enc_cset_eq(res));
 9264 
 9265   ins_pipe(pipe_slow);
 9266 %}
 9267 
 9268 // alternative CompareAndSwapX when we are eliding barriers
 9269 
 9270 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9271 
 9272   predicate(needs_acquiring_load_exclusive(n));
 9273   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9274   ins_cost(VOLATILE_REF_COST);
 9275 
 9276   effect(KILL cr);
 9277 
 9278   format %{
 9279     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9280     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9281   %}
 9282 
 9283   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9284             aarch64_enc_cset_eq(res));
 9285 
 9286   ins_pipe(pipe_slow);
 9287 %}
 9288 
 9289 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9290 
 9291   predicate(needs_acquiring_load_exclusive(n));
 9292   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9293   ins_cost(VOLATILE_REF_COST);
 9294 
 9295   effect(KILL cr);
 9296 
 9297   format %{
 9298     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9299     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9300   %}
 9301 
 9302   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9303             aarch64_enc_cset_eq(res));
 9304 
 9305   ins_pipe(pipe_slow);
 9306 %}
 9307 
 9308 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9309 
 9310   predicate(needs_acquiring_load_exclusive(n));
 9311   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9312   ins_cost(VOLATILE_REF_COST);
 9313 
 9314   effect(KILL cr);
 9315 
 9316  format %{
 9317     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9318     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9319  %}
 9320 
 9321  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9322             aarch64_enc_cset_eq(res));
 9323 
 9324   ins_pipe(pipe_slow);
 9325 %}
 9326 
 9327 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9328 
 9329   predicate(needs_acquiring_load_exclusive(n));
 9330   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9331   ins_cost(VOLATILE_REF_COST);
 9332 
 9333   effect(KILL cr);
 9334 
 9335  format %{
 9336     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9337     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9338  %}
 9339 
 9340  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9341             aarch64_enc_cset_eq(res));
 9342 
 9343   ins_pipe(pipe_slow);
 9344 %}
 9345 
 9346 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9347 
 9348   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9349   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9350   ins_cost(VOLATILE_REF_COST);
 9351 
 9352   effect(KILL cr);
 9353 
 9354  format %{
 9355     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9356     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9357  %}
 9358 
 9359  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9360             aarch64_enc_cset_eq(res));
 9361 
 9362   ins_pipe(pipe_slow);
 9363 %}
 9364 
 9365 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9366 
 9367   predicate(needs_acquiring_load_exclusive(n));
 9368   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9369   ins_cost(VOLATILE_REF_COST);
 9370 
 9371   effect(KILL cr);
 9372 
 9373  format %{
 9374     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9375     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9376  %}
 9377 
 9378  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9379             aarch64_enc_cset_eq(res));
 9380 
 9381   ins_pipe(pipe_slow);
 9382 %}
 9383 
 9384 
 9385 // ---------------------------------------------------------------------
 9386 
 9387 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9388 
 9389 // Sundry CAS operations.  Note that release is always true,
 9390 // regardless of the memory ordering of the CAS.  This is because we
 9391 // need the volatile case to be sequentially consistent but there is
 9392 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9393 // can't check the type of memory ordering here, so we always emit a
 9394 // STLXR.
 9395 
 9396 // This section is generated from aarch64_ad_cas.m4
 9397 
 9398 
 9399 
 9400 // This pattern is generated automatically from cas.m4.
 9401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9402 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9403 
 9404   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9405   ins_cost(2 * VOLATILE_REF_COST);
 9406   effect(TEMP_DEF res, KILL cr);
 9407   format %{
 9408     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9409   %}
 9410   ins_encode %{
 9411     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9412                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9413                /*weak*/ false, $res$$Register);
 9414     __ sxtbw($res$$Register, $res$$Register);
 9415   %}
 9416   ins_pipe(pipe_slow);
 9417 %}
 9418 
 9419 // This pattern is generated automatically from cas.m4.
 9420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9421 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9422 
 9423   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9424   ins_cost(2 * VOLATILE_REF_COST);
 9425   effect(TEMP_DEF res, KILL cr);
 9426   format %{
 9427     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9428   %}
 9429   ins_encode %{
 9430     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9431                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9432                /*weak*/ false, $res$$Register);
 9433     __ sxthw($res$$Register, $res$$Register);
 9434   %}
 9435   ins_pipe(pipe_slow);
 9436 %}
 9437 
 9438 // This pattern is generated automatically from cas.m4.
 9439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9440 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9441 
 9442   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9443   ins_cost(2 * VOLATILE_REF_COST);
 9444   effect(TEMP_DEF res, KILL cr);
 9445   format %{
 9446     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9447   %}
 9448   ins_encode %{
 9449     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9450                Assembler::word, /*acquire*/ false, /*release*/ true,
 9451                /*weak*/ false, $res$$Register);
 9452   %}
 9453   ins_pipe(pipe_slow);
 9454 %}
 9455 
 9456 // This pattern is generated automatically from cas.m4.
 9457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9458 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9459 
 9460   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9461   ins_cost(2 * VOLATILE_REF_COST);
 9462   effect(TEMP_DEF res, KILL cr);
 9463   format %{
 9464     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9465   %}
 9466   ins_encode %{
 9467     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9468                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9469                /*weak*/ false, $res$$Register);
 9470   %}
 9471   ins_pipe(pipe_slow);
 9472 %}
 9473 
 9474 // This pattern is generated automatically from cas.m4.
 9475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9476 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9477 
 9478   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9479   ins_cost(2 * VOLATILE_REF_COST);
 9480   effect(TEMP_DEF res, KILL cr);
 9481   format %{
 9482     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9483   %}
 9484   ins_encode %{
 9485     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9486                Assembler::word, /*acquire*/ false, /*release*/ true,
 9487                /*weak*/ false, $res$$Register);
 9488   %}
 9489   ins_pipe(pipe_slow);
 9490 %}
 9491 
 9492 // This pattern is generated automatically from cas.m4.
 9493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9494 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9495   predicate(n->as_LoadStore()->barrier_data() == 0);
 9496   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9497   ins_cost(2 * VOLATILE_REF_COST);
 9498   effect(TEMP_DEF res, KILL cr);
 9499   format %{
 9500     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9501   %}
 9502   ins_encode %{
 9503     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9504                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9505                /*weak*/ false, $res$$Register);
 9506   %}
 9507   ins_pipe(pipe_slow);
 9508 %}
 9509 
 9510 // This pattern is generated automatically from cas.m4.
 9511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9512 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9513   predicate(needs_acquiring_load_exclusive(n));
 9514   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9515   ins_cost(VOLATILE_REF_COST);
 9516   effect(TEMP_DEF res, KILL cr);
 9517   format %{
 9518     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9519   %}
 9520   ins_encode %{
 9521     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9522                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9523                /*weak*/ false, $res$$Register);
 9524     __ sxtbw($res$$Register, $res$$Register);
 9525   %}
 9526   ins_pipe(pipe_slow);
 9527 %}
 9528 
 9529 // This pattern is generated automatically from cas.m4.
 9530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9531 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9532   predicate(needs_acquiring_load_exclusive(n));
 9533   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9534   ins_cost(VOLATILE_REF_COST);
 9535   effect(TEMP_DEF res, KILL cr);
 9536   format %{
 9537     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9538   %}
 9539   ins_encode %{
 9540     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9541                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9542                /*weak*/ false, $res$$Register);
 9543     __ sxthw($res$$Register, $res$$Register);
 9544   %}
 9545   ins_pipe(pipe_slow);
 9546 %}
 9547 
 9548 // This pattern is generated automatically from cas.m4.
 9549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9550 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9551   predicate(needs_acquiring_load_exclusive(n));
 9552   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9553   ins_cost(VOLATILE_REF_COST);
 9554   effect(TEMP_DEF res, KILL cr);
 9555   format %{
 9556     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9557   %}
 9558   ins_encode %{
 9559     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9560                Assembler::word, /*acquire*/ true, /*release*/ true,
 9561                /*weak*/ false, $res$$Register);
 9562   %}
 9563   ins_pipe(pipe_slow);
 9564 %}
 9565 
 9566 // This pattern is generated automatically from cas.m4.
 9567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9568 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9569   predicate(needs_acquiring_load_exclusive(n));
 9570   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9571   ins_cost(VOLATILE_REF_COST);
 9572   effect(TEMP_DEF res, KILL cr);
 9573   format %{
 9574     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9575   %}
 9576   ins_encode %{
 9577     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9578                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9579                /*weak*/ false, $res$$Register);
 9580   %}
 9581   ins_pipe(pipe_slow);
 9582 %}
 9583 
 9584 // This pattern is generated automatically from cas.m4.
 9585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9586 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9587   predicate(needs_acquiring_load_exclusive(n));
 9588   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9589   ins_cost(VOLATILE_REF_COST);
 9590   effect(TEMP_DEF res, KILL cr);
 9591   format %{
 9592     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9593   %}
 9594   ins_encode %{
 9595     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9596                Assembler::word, /*acquire*/ true, /*release*/ true,
 9597                /*weak*/ false, $res$$Register);
 9598   %}
 9599   ins_pipe(pipe_slow);
 9600 %}
 9601 
 9602 // This pattern is generated automatically from cas.m4.
 9603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9604 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9605   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9606   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9607   ins_cost(VOLATILE_REF_COST);
 9608   effect(TEMP_DEF res, KILL cr);
 9609   format %{
 9610     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9611   %}
 9612   ins_encode %{
 9613     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9614                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9615                /*weak*/ false, $res$$Register);
 9616   %}
 9617   ins_pipe(pipe_slow);
 9618 %}
 9619 
 9620 // This pattern is generated automatically from cas.m4.
 9621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9622 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9623 
 9624   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9625   ins_cost(2 * VOLATILE_REF_COST);
 9626   effect(KILL cr);
 9627   format %{
 9628     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9629     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9630   %}
 9631   ins_encode %{
 9632     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9633                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9634                /*weak*/ true, noreg);
 9635     __ csetw($res$$Register, Assembler::EQ);
 9636   %}
 9637   ins_pipe(pipe_slow);
 9638 %}
 9639 
 9640 // This pattern is generated automatically from cas.m4.
 9641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9642 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9643 
 9644   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9645   ins_cost(2 * VOLATILE_REF_COST);
 9646   effect(KILL cr);
 9647   format %{
 9648     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9649     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9650   %}
 9651   ins_encode %{
 9652     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9653                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9654                /*weak*/ true, noreg);
 9655     __ csetw($res$$Register, Assembler::EQ);
 9656   %}
 9657   ins_pipe(pipe_slow);
 9658 %}
 9659 
 9660 // This pattern is generated automatically from cas.m4.
 9661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9662 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9663 
 9664   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9665   ins_cost(2 * VOLATILE_REF_COST);
 9666   effect(KILL cr);
 9667   format %{
 9668     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9669     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9670   %}
 9671   ins_encode %{
 9672     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9673                Assembler::word, /*acquire*/ false, /*release*/ true,
 9674                /*weak*/ true, noreg);
 9675     __ csetw($res$$Register, Assembler::EQ);
 9676   %}
 9677   ins_pipe(pipe_slow);
 9678 %}
 9679 
 9680 // This pattern is generated automatically from cas.m4.
 9681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9682 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9683 
 9684   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9685   ins_cost(2 * VOLATILE_REF_COST);
 9686   effect(KILL cr);
 9687   format %{
 9688     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9689     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9690   %}
 9691   ins_encode %{
 9692     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9693                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9694                /*weak*/ true, noreg);
 9695     __ csetw($res$$Register, Assembler::EQ);
 9696   %}
 9697   ins_pipe(pipe_slow);
 9698 %}
 9699 
 9700 // This pattern is generated automatically from cas.m4.
 9701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9702 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9703 
 9704   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9705   ins_cost(2 * VOLATILE_REF_COST);
 9706   effect(KILL cr);
 9707   format %{
 9708     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9709     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9710   %}
 9711   ins_encode %{
 9712     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9713                Assembler::word, /*acquire*/ false, /*release*/ true,
 9714                /*weak*/ true, noreg);
 9715     __ csetw($res$$Register, Assembler::EQ);
 9716   %}
 9717   ins_pipe(pipe_slow);
 9718 %}
 9719 
 9720 // This pattern is generated automatically from cas.m4.
 9721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9722 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9723   predicate(n->as_LoadStore()->barrier_data() == 0);
 9724   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9725   ins_cost(2 * VOLATILE_REF_COST);
 9726   effect(KILL cr);
 9727   format %{
 9728     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9729     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9730   %}
 9731   ins_encode %{
 9732     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9733                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9734                /*weak*/ true, noreg);
 9735     __ csetw($res$$Register, Assembler::EQ);
 9736   %}
 9737   ins_pipe(pipe_slow);
 9738 %}
 9739 
 9740 // This pattern is generated automatically from cas.m4.
 9741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9742 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9743   predicate(needs_acquiring_load_exclusive(n));
 9744   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9745   ins_cost(VOLATILE_REF_COST);
 9746   effect(KILL cr);
 9747   format %{
 9748     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9749     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9750   %}
 9751   ins_encode %{
 9752     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9753                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9754                /*weak*/ true, noreg);
 9755     __ csetw($res$$Register, Assembler::EQ);
 9756   %}
 9757   ins_pipe(pipe_slow);
 9758 %}
 9759 
 9760 // This pattern is generated automatically from cas.m4.
 9761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9762 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9763   predicate(needs_acquiring_load_exclusive(n));
 9764   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9765   ins_cost(VOLATILE_REF_COST);
 9766   effect(KILL cr);
 9767   format %{
 9768     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9769     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9770   %}
 9771   ins_encode %{
 9772     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9773                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9774                /*weak*/ true, noreg);
 9775     __ csetw($res$$Register, Assembler::EQ);
 9776   %}
 9777   ins_pipe(pipe_slow);
 9778 %}
 9779 
 9780 // This pattern is generated automatically from cas.m4.
 9781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9782 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9783   predicate(needs_acquiring_load_exclusive(n));
 9784   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9785   ins_cost(VOLATILE_REF_COST);
 9786   effect(KILL cr);
 9787   format %{
 9788     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9789     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9790   %}
 9791   ins_encode %{
 9792     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9793                Assembler::word, /*acquire*/ true, /*release*/ true,
 9794                /*weak*/ true, noreg);
 9795     __ csetw($res$$Register, Assembler::EQ);
 9796   %}
 9797   ins_pipe(pipe_slow);
 9798 %}
 9799 
 9800 // This pattern is generated automatically from cas.m4.
 9801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9802 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9803   predicate(needs_acquiring_load_exclusive(n));
 9804   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9805   ins_cost(VOLATILE_REF_COST);
 9806   effect(KILL cr);
 9807   format %{
 9808     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9809     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9810   %}
 9811   ins_encode %{
 9812     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9813                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9814                /*weak*/ true, noreg);
 9815     __ csetw($res$$Register, Assembler::EQ);
 9816   %}
 9817   ins_pipe(pipe_slow);
 9818 %}
 9819 
 9820 // This pattern is generated automatically from cas.m4.
 9821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9822 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9823   predicate(needs_acquiring_load_exclusive(n));
 9824   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9825   ins_cost(VOLATILE_REF_COST);
 9826   effect(KILL cr);
 9827   format %{
 9828     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9829     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9830   %}
 9831   ins_encode %{
 9832     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9833                Assembler::word, /*acquire*/ true, /*release*/ true,
 9834                /*weak*/ true, noreg);
 9835     __ csetw($res$$Register, Assembler::EQ);
 9836   %}
 9837   ins_pipe(pipe_slow);
 9838 %}
 9839 
 9840 // This pattern is generated automatically from cas.m4.
 9841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9842 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9843   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9844   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9845   ins_cost(VOLATILE_REF_COST);
 9846   effect(KILL cr);
 9847   format %{
 9848     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9849     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9850   %}
 9851   ins_encode %{
 9852     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9853                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9854                /*weak*/ true, noreg);
 9855     __ csetw($res$$Register, Assembler::EQ);
 9856   %}
 9857   ins_pipe(pipe_slow);
 9858 %}
 9859 
 9860 // END This section of the file is automatically generated. Do not edit --------------
 9861 // ---------------------------------------------------------------------
 9862 
 9863 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9864   match(Set prev (GetAndSetI mem newv));
 9865   ins_cost(2 * VOLATILE_REF_COST);
 9866   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9867   ins_encode %{
 9868     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9869   %}
 9870   ins_pipe(pipe_serial);
 9871 %}
 9872 
 9873 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9874   match(Set prev (GetAndSetL mem newv));
 9875   ins_cost(2 * VOLATILE_REF_COST);
 9876   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9877   ins_encode %{
 9878     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9879   %}
 9880   ins_pipe(pipe_serial);
 9881 %}
 9882 
 9883 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9884   match(Set prev (GetAndSetN mem newv));
 9885   ins_cost(2 * VOLATILE_REF_COST);
 9886   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9887   ins_encode %{
 9888     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9889   %}
 9890   ins_pipe(pipe_serial);
 9891 %}
 9892 
 9893 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9894   predicate(n->as_LoadStore()->barrier_data() == 0);
 9895   match(Set prev (GetAndSetP mem newv));
 9896   ins_cost(2 * VOLATILE_REF_COST);
 9897   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9898   ins_encode %{
 9899     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9900   %}
 9901   ins_pipe(pipe_serial);
 9902 %}
 9903 
 9904 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9905   predicate(needs_acquiring_load_exclusive(n));
 9906   match(Set prev (GetAndSetI mem newv));
 9907   ins_cost(VOLATILE_REF_COST);
 9908   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9909   ins_encode %{
 9910     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9911   %}
 9912   ins_pipe(pipe_serial);
 9913 %}
 9914 
 9915 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9916   predicate(needs_acquiring_load_exclusive(n));
 9917   match(Set prev (GetAndSetL mem newv));
 9918   ins_cost(VOLATILE_REF_COST);
 9919   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9920   ins_encode %{
 9921     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9922   %}
 9923   ins_pipe(pipe_serial);
 9924 %}
 9925 
 9926 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9927   predicate(needs_acquiring_load_exclusive(n));
 9928   match(Set prev (GetAndSetN mem newv));
 9929   ins_cost(VOLATILE_REF_COST);
 9930   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9931   ins_encode %{
 9932     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9933   %}
 9934   ins_pipe(pipe_serial);
 9935 %}
 9936 
 9937 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9938   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9939   match(Set prev (GetAndSetP mem newv));
 9940   ins_cost(VOLATILE_REF_COST);
 9941   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9942   ins_encode %{
 9943     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9944   %}
 9945   ins_pipe(pipe_serial);
 9946 %}
 9947 
 9948 
 9949 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9950   match(Set newval (GetAndAddL mem incr));
 9951   ins_cost(2 * VOLATILE_REF_COST + 1);
 9952   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9953   ins_encode %{
 9954     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9955   %}
 9956   ins_pipe(pipe_serial);
 9957 %}
 9958 
 9959 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9960   predicate(n->as_LoadStore()->result_not_used());
 9961   match(Set dummy (GetAndAddL mem incr));
 9962   ins_cost(2 * VOLATILE_REF_COST);
 9963   format %{ "get_and_addL [$mem], $incr" %}
 9964   ins_encode %{
 9965     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9966   %}
 9967   ins_pipe(pipe_serial);
 9968 %}
 9969 
 9970 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9971   match(Set newval (GetAndAddL mem incr));
 9972   ins_cost(2 * VOLATILE_REF_COST + 1);
 9973   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9974   ins_encode %{
 9975     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9976   %}
 9977   ins_pipe(pipe_serial);
 9978 %}
 9979 
 9980 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9981   predicate(n->as_LoadStore()->result_not_used());
 9982   match(Set dummy (GetAndAddL mem incr));
 9983   ins_cost(2 * VOLATILE_REF_COST);
 9984   format %{ "get_and_addL [$mem], $incr" %}
 9985   ins_encode %{
 9986     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9987   %}
 9988   ins_pipe(pipe_serial);
 9989 %}
 9990 
 9991 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9992   match(Set newval (GetAndAddI mem incr));
 9993   ins_cost(2 * VOLATILE_REF_COST + 1);
 9994   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9995   ins_encode %{
 9996     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9997   %}
 9998   ins_pipe(pipe_serial);
 9999 %}
10000 
10001 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10002   predicate(n->as_LoadStore()->result_not_used());
10003   match(Set dummy (GetAndAddI mem incr));
10004   ins_cost(2 * VOLATILE_REF_COST);
10005   format %{ "get_and_addI [$mem], $incr" %}
10006   ins_encode %{
10007     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10008   %}
10009   ins_pipe(pipe_serial);
10010 %}
10011 
10012 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10013   match(Set newval (GetAndAddI mem incr));
10014   ins_cost(2 * VOLATILE_REF_COST + 1);
10015   format %{ "get_and_addI $newval, [$mem], $incr" %}
10016   ins_encode %{
10017     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10018   %}
10019   ins_pipe(pipe_serial);
10020 %}
10021 
10022 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10023   predicate(n->as_LoadStore()->result_not_used());
10024   match(Set dummy (GetAndAddI mem incr));
10025   ins_cost(2 * VOLATILE_REF_COST);
10026   format %{ "get_and_addI [$mem], $incr" %}
10027   ins_encode %{
10028     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10029   %}
10030   ins_pipe(pipe_serial);
10031 %}
10032 
10033 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10034   predicate(needs_acquiring_load_exclusive(n));
10035   match(Set newval (GetAndAddL mem incr));
10036   ins_cost(VOLATILE_REF_COST + 1);
10037   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10038   ins_encode %{
10039     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10040   %}
10041   ins_pipe(pipe_serial);
10042 %}
10043 
10044 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10045   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10046   match(Set dummy (GetAndAddL mem incr));
10047   ins_cost(VOLATILE_REF_COST);
10048   format %{ "get_and_addL_acq [$mem], $incr" %}
10049   ins_encode %{
10050     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10051   %}
10052   ins_pipe(pipe_serial);
10053 %}
10054 
10055 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10056   predicate(needs_acquiring_load_exclusive(n));
10057   match(Set newval (GetAndAddL mem incr));
10058   ins_cost(VOLATILE_REF_COST + 1);
10059   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10060   ins_encode %{
10061     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10062   %}
10063   ins_pipe(pipe_serial);
10064 %}
10065 
10066 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10067   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10068   match(Set dummy (GetAndAddL mem incr));
10069   ins_cost(VOLATILE_REF_COST);
10070   format %{ "get_and_addL_acq [$mem], $incr" %}
10071   ins_encode %{
10072     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10073   %}
10074   ins_pipe(pipe_serial);
10075 %}
10076 
10077 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10078   predicate(needs_acquiring_load_exclusive(n));
10079   match(Set newval (GetAndAddI mem incr));
10080   ins_cost(VOLATILE_REF_COST + 1);
10081   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10082   ins_encode %{
10083     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10084   %}
10085   ins_pipe(pipe_serial);
10086 %}
10087 
10088 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10089   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10090   match(Set dummy (GetAndAddI mem incr));
10091   ins_cost(VOLATILE_REF_COST);
10092   format %{ "get_and_addI_acq [$mem], $incr" %}
10093   ins_encode %{
10094     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10095   %}
10096   ins_pipe(pipe_serial);
10097 %}
10098 
10099 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10100   predicate(needs_acquiring_load_exclusive(n));
10101   match(Set newval (GetAndAddI mem incr));
10102   ins_cost(VOLATILE_REF_COST + 1);
10103   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10104   ins_encode %{
10105     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10106   %}
10107   ins_pipe(pipe_serial);
10108 %}
10109 
10110 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10111   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10112   match(Set dummy (GetAndAddI mem incr));
10113   ins_cost(VOLATILE_REF_COST);
10114   format %{ "get_and_addI_acq [$mem], $incr" %}
10115   ins_encode %{
10116     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10117   %}
10118   ins_pipe(pipe_serial);
10119 %}
10120 
10121 // Manifest a CmpL result in an integer register.
10122 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10123 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10124 %{
10125   match(Set dst (CmpL3 src1 src2));
10126   effect(KILL flags);
10127 
10128   ins_cost(INSN_COST * 6);
10129   format %{
10130       "cmp $src1, $src2"
10131       "csetw $dst, ne"
10132       "cnegw $dst, lt"
10133   %}
10134   // format %{ "CmpL3 $dst, $src1, $src2" %}
10135   ins_encode %{
10136     __ cmp($src1$$Register, $src2$$Register);
10137     __ csetw($dst$$Register, Assembler::NE);
10138     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10139   %}
10140 
10141   ins_pipe(pipe_class_default);
10142 %}
10143 
10144 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10145 %{
10146   match(Set dst (CmpL3 src1 src2));
10147   effect(KILL flags);
10148 
10149   ins_cost(INSN_COST * 6);
10150   format %{
10151       "cmp $src1, $src2"
10152       "csetw $dst, ne"
10153       "cnegw $dst, lt"
10154   %}
10155   ins_encode %{
10156     int32_t con = (int32_t)$src2$$constant;
10157      if (con < 0) {
10158       __ adds(zr, $src1$$Register, -con);
10159     } else {
10160       __ subs(zr, $src1$$Register, con);
10161     }
10162     __ csetw($dst$$Register, Assembler::NE);
10163     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10164   %}
10165 
10166   ins_pipe(pipe_class_default);
10167 %}
10168 
10169 // ============================================================================
10170 // Conditional Move Instructions
10171 
10172 // n.b. we have identical rules for both a signed compare op (cmpOp)
10173 // and an unsigned compare op (cmpOpU). it would be nice if we could
10174 // define an op class which merged both inputs and use it to type the
10175 // argument to a single rule. unfortunatelyt his fails because the
10176 // opclass does not live up to the COND_INTER interface of its
10177 // component operands. When the generic code tries to negate the
10178 // operand it ends up running the generci Machoper::negate method
10179 // which throws a ShouldNotHappen. So, we have to provide two flavours
10180 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10181 
10182 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10183   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10184 
10185   ins_cost(INSN_COST * 2);
10186   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10187 
10188   ins_encode %{
10189     __ cselw(as_Register($dst$$reg),
10190              as_Register($src2$$reg),
10191              as_Register($src1$$reg),
10192              (Assembler::Condition)$cmp$$cmpcode);
10193   %}
10194 
10195   ins_pipe(icond_reg_reg);
10196 %}
10197 
10198 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10199   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10200 
10201   ins_cost(INSN_COST * 2);
10202   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10203 
10204   ins_encode %{
10205     __ cselw(as_Register($dst$$reg),
10206              as_Register($src2$$reg),
10207              as_Register($src1$$reg),
10208              (Assembler::Condition)$cmp$$cmpcode);
10209   %}
10210 
10211   ins_pipe(icond_reg_reg);
10212 %}
10213 
10214 // special cases where one arg is zero
10215 
10216 // n.b. this is selected in preference to the rule above because it
10217 // avoids loading constant 0 into a source register
10218 
10219 // TODO
10220 // we ought only to be able to cull one of these variants as the ideal
10221 // transforms ought always to order the zero consistently (to left/right?)
10222 
10223 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10224   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10225 
10226   ins_cost(INSN_COST * 2);
10227   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10228 
10229   ins_encode %{
10230     __ cselw(as_Register($dst$$reg),
10231              as_Register($src$$reg),
10232              zr,
10233              (Assembler::Condition)$cmp$$cmpcode);
10234   %}
10235 
10236   ins_pipe(icond_reg);
10237 %}
10238 
10239 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10240   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10241 
10242   ins_cost(INSN_COST * 2);
10243   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10244 
10245   ins_encode %{
10246     __ cselw(as_Register($dst$$reg),
10247              as_Register($src$$reg),
10248              zr,
10249              (Assembler::Condition)$cmp$$cmpcode);
10250   %}
10251 
10252   ins_pipe(icond_reg);
10253 %}
10254 
10255 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10256   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10257 
10258   ins_cost(INSN_COST * 2);
10259   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10260 
10261   ins_encode %{
10262     __ cselw(as_Register($dst$$reg),
10263              zr,
10264              as_Register($src$$reg),
10265              (Assembler::Condition)$cmp$$cmpcode);
10266   %}
10267 
10268   ins_pipe(icond_reg);
10269 %}
10270 
10271 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10272   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10273 
10274   ins_cost(INSN_COST * 2);
10275   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10276 
10277   ins_encode %{
10278     __ cselw(as_Register($dst$$reg),
10279              zr,
10280              as_Register($src$$reg),
10281              (Assembler::Condition)$cmp$$cmpcode);
10282   %}
10283 
10284   ins_pipe(icond_reg);
10285 %}
10286 
10287 // special case for creating a boolean 0 or 1
10288 
10289 // n.b. this is selected in preference to the rule above because it
10290 // avoids loading constants 0 and 1 into a source register
10291 
10292 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10293   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10294 
10295   ins_cost(INSN_COST * 2);
10296   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10297 
10298   ins_encode %{
10299     // equivalently
10300     // cset(as_Register($dst$$reg),
10301     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10302     __ csincw(as_Register($dst$$reg),
10303              zr,
10304              zr,
10305              (Assembler::Condition)$cmp$$cmpcode);
10306   %}
10307 
10308   ins_pipe(icond_none);
10309 %}
10310 
10311 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10312   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10313 
10314   ins_cost(INSN_COST * 2);
10315   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10316 
10317   ins_encode %{
10318     // equivalently
10319     // cset(as_Register($dst$$reg),
10320     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10321     __ csincw(as_Register($dst$$reg),
10322              zr,
10323              zr,
10324              (Assembler::Condition)$cmp$$cmpcode);
10325   %}
10326 
10327   ins_pipe(icond_none);
10328 %}
10329 
10330 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10331   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10332 
10333   ins_cost(INSN_COST * 2);
10334   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10335 
10336   ins_encode %{
10337     __ csel(as_Register($dst$$reg),
10338             as_Register($src2$$reg),
10339             as_Register($src1$$reg),
10340             (Assembler::Condition)$cmp$$cmpcode);
10341   %}
10342 
10343   ins_pipe(icond_reg_reg);
10344 %}
10345 
10346 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10347   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10348 
10349   ins_cost(INSN_COST * 2);
10350   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10351 
10352   ins_encode %{
10353     __ csel(as_Register($dst$$reg),
10354             as_Register($src2$$reg),
10355             as_Register($src1$$reg),
10356             (Assembler::Condition)$cmp$$cmpcode);
10357   %}
10358 
10359   ins_pipe(icond_reg_reg);
10360 %}
10361 
10362 // special cases where one arg is zero
10363 
10364 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10365   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10366 
10367   ins_cost(INSN_COST * 2);
10368   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10369 
10370   ins_encode %{
10371     __ csel(as_Register($dst$$reg),
10372             zr,
10373             as_Register($src$$reg),
10374             (Assembler::Condition)$cmp$$cmpcode);
10375   %}
10376 
10377   ins_pipe(icond_reg);
10378 %}
10379 
10380 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10381   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10382 
10383   ins_cost(INSN_COST * 2);
10384   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10385 
10386   ins_encode %{
10387     __ csel(as_Register($dst$$reg),
10388             zr,
10389             as_Register($src$$reg),
10390             (Assembler::Condition)$cmp$$cmpcode);
10391   %}
10392 
10393   ins_pipe(icond_reg);
10394 %}
10395 
10396 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10397   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10398 
10399   ins_cost(INSN_COST * 2);
10400   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10401 
10402   ins_encode %{
10403     __ csel(as_Register($dst$$reg),
10404             as_Register($src$$reg),
10405             zr,
10406             (Assembler::Condition)$cmp$$cmpcode);
10407   %}
10408 
10409   ins_pipe(icond_reg);
10410 %}
10411 
10412 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10413   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10414 
10415   ins_cost(INSN_COST * 2);
10416   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10417 
10418   ins_encode %{
10419     __ csel(as_Register($dst$$reg),
10420             as_Register($src$$reg),
10421             zr,
10422             (Assembler::Condition)$cmp$$cmpcode);
10423   %}
10424 
10425   ins_pipe(icond_reg);
10426 %}
10427 
10428 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10429   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10430 
10431   ins_cost(INSN_COST * 2);
10432   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10433 
10434   ins_encode %{
10435     __ csel(as_Register($dst$$reg),
10436             as_Register($src2$$reg),
10437             as_Register($src1$$reg),
10438             (Assembler::Condition)$cmp$$cmpcode);
10439   %}
10440 
10441   ins_pipe(icond_reg_reg);
10442 %}
10443 
10444 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10445   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10446 
10447   ins_cost(INSN_COST * 2);
10448   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10449 
10450   ins_encode %{
10451     __ csel(as_Register($dst$$reg),
10452             as_Register($src2$$reg),
10453             as_Register($src1$$reg),
10454             (Assembler::Condition)$cmp$$cmpcode);
10455   %}
10456 
10457   ins_pipe(icond_reg_reg);
10458 %}
10459 
10460 // special cases where one arg is zero
10461 
10462 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10463   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10464 
10465   ins_cost(INSN_COST * 2);
10466   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10467 
10468   ins_encode %{
10469     __ csel(as_Register($dst$$reg),
10470             zr,
10471             as_Register($src$$reg),
10472             (Assembler::Condition)$cmp$$cmpcode);
10473   %}
10474 
10475   ins_pipe(icond_reg);
10476 %}
10477 
10478 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10479   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10480 
10481   ins_cost(INSN_COST * 2);
10482   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10483 
10484   ins_encode %{
10485     __ csel(as_Register($dst$$reg),
10486             zr,
10487             as_Register($src$$reg),
10488             (Assembler::Condition)$cmp$$cmpcode);
10489   %}
10490 
10491   ins_pipe(icond_reg);
10492 %}
10493 
10494 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10495   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10496 
10497   ins_cost(INSN_COST * 2);
10498   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10499 
10500   ins_encode %{
10501     __ csel(as_Register($dst$$reg),
10502             as_Register($src$$reg),
10503             zr,
10504             (Assembler::Condition)$cmp$$cmpcode);
10505   %}
10506 
10507   ins_pipe(icond_reg);
10508 %}
10509 
10510 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10511   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10512 
10513   ins_cost(INSN_COST * 2);
10514   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10515 
10516   ins_encode %{
10517     __ csel(as_Register($dst$$reg),
10518             as_Register($src$$reg),
10519             zr,
10520             (Assembler::Condition)$cmp$$cmpcode);
10521   %}
10522 
10523   ins_pipe(icond_reg);
10524 %}
10525 
10526 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10527   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10528 
10529   ins_cost(INSN_COST * 2);
10530   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10531 
10532   ins_encode %{
10533     __ cselw(as_Register($dst$$reg),
10534              as_Register($src2$$reg),
10535              as_Register($src1$$reg),
10536              (Assembler::Condition)$cmp$$cmpcode);
10537   %}
10538 
10539   ins_pipe(icond_reg_reg);
10540 %}
10541 
10542 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10543   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10544 
10545   ins_cost(INSN_COST * 2);
10546   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10547 
10548   ins_encode %{
10549     __ cselw(as_Register($dst$$reg),
10550              as_Register($src2$$reg),
10551              as_Register($src1$$reg),
10552              (Assembler::Condition)$cmp$$cmpcode);
10553   %}
10554 
10555   ins_pipe(icond_reg_reg);
10556 %}
10557 
10558 // special cases where one arg is zero
10559 
10560 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10561   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10562 
10563   ins_cost(INSN_COST * 2);
10564   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10565 
10566   ins_encode %{
10567     __ cselw(as_Register($dst$$reg),
10568              zr,
10569              as_Register($src$$reg),
10570              (Assembler::Condition)$cmp$$cmpcode);
10571   %}
10572 
10573   ins_pipe(icond_reg);
10574 %}
10575 
10576 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10577   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10578 
10579   ins_cost(INSN_COST * 2);
10580   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10581 
10582   ins_encode %{
10583     __ cselw(as_Register($dst$$reg),
10584              zr,
10585              as_Register($src$$reg),
10586              (Assembler::Condition)$cmp$$cmpcode);
10587   %}
10588 
10589   ins_pipe(icond_reg);
10590 %}
10591 
10592 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10593   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10594 
10595   ins_cost(INSN_COST * 2);
10596   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10597 
10598   ins_encode %{
10599     __ cselw(as_Register($dst$$reg),
10600              as_Register($src$$reg),
10601              zr,
10602              (Assembler::Condition)$cmp$$cmpcode);
10603   %}
10604 
10605   ins_pipe(icond_reg);
10606 %}
10607 
10608 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10609   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10610 
10611   ins_cost(INSN_COST * 2);
10612   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10613 
10614   ins_encode %{
10615     __ cselw(as_Register($dst$$reg),
10616              as_Register($src$$reg),
10617              zr,
10618              (Assembler::Condition)$cmp$$cmpcode);
10619   %}
10620 
10621   ins_pipe(icond_reg);
10622 %}
10623 
10624 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10625 %{
10626   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10627 
10628   ins_cost(INSN_COST * 3);
10629 
10630   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10631   ins_encode %{
10632     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10633     __ fcsels(as_FloatRegister($dst$$reg),
10634               as_FloatRegister($src2$$reg),
10635               as_FloatRegister($src1$$reg),
10636               cond);
10637   %}
10638 
10639   ins_pipe(fp_cond_reg_reg_s);
10640 %}
10641 
10642 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10643 %{
10644   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10645 
10646   ins_cost(INSN_COST * 3);
10647 
10648   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10649   ins_encode %{
10650     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10651     __ fcsels(as_FloatRegister($dst$$reg),
10652               as_FloatRegister($src2$$reg),
10653               as_FloatRegister($src1$$reg),
10654               cond);
10655   %}
10656 
10657   ins_pipe(fp_cond_reg_reg_s);
10658 %}
10659 
10660 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10661 %{
10662   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10663 
10664   ins_cost(INSN_COST * 3);
10665 
10666   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10667   ins_encode %{
10668     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10669     __ fcseld(as_FloatRegister($dst$$reg),
10670               as_FloatRegister($src2$$reg),
10671               as_FloatRegister($src1$$reg),
10672               cond);
10673   %}
10674 
10675   ins_pipe(fp_cond_reg_reg_d);
10676 %}
10677 
10678 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10679 %{
10680   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10681 
10682   ins_cost(INSN_COST * 3);
10683 
10684   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10685   ins_encode %{
10686     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10687     __ fcseld(as_FloatRegister($dst$$reg),
10688               as_FloatRegister($src2$$reg),
10689               as_FloatRegister($src1$$reg),
10690               cond);
10691   %}
10692 
10693   ins_pipe(fp_cond_reg_reg_d);
10694 %}
10695 
10696 // ============================================================================
10697 // Arithmetic Instructions
10698 //
10699 
10700 // Integer Addition
10701 
10702 // TODO
10703 // these currently employ operations which do not set CR and hence are
10704 // not flagged as killing CR but we would like to isolate the cases
10705 // where we want to set flags from those where we don't. need to work
10706 // out how to do that.
10707 
10708 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10709   match(Set dst (AddI src1 src2));
10710 
10711   ins_cost(INSN_COST);
10712   format %{ "addw  $dst, $src1, $src2" %}
10713 
10714   ins_encode %{
10715     __ addw(as_Register($dst$$reg),
10716             as_Register($src1$$reg),
10717             as_Register($src2$$reg));
10718   %}
10719 
10720   ins_pipe(ialu_reg_reg);
10721 %}
10722 
10723 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10724   match(Set dst (AddI src1 src2));
10725 
10726   ins_cost(INSN_COST);
10727   format %{ "addw $dst, $src1, $src2" %}
10728 
10729   // use opcode to indicate that this is an add not a sub
10730   opcode(0x0);
10731 
10732   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10733 
10734   ins_pipe(ialu_reg_imm);
10735 %}
10736 
10737 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10738   match(Set dst (AddI (ConvL2I src1) src2));
10739 
10740   ins_cost(INSN_COST);
10741   format %{ "addw $dst, $src1, $src2" %}
10742 
10743   // use opcode to indicate that this is an add not a sub
10744   opcode(0x0);
10745 
10746   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10747 
10748   ins_pipe(ialu_reg_imm);
10749 %}
10750 
10751 // Pointer Addition
10752 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10753   match(Set dst (AddP src1 src2));
10754 
10755   ins_cost(INSN_COST);
10756   format %{ "add $dst, $src1, $src2\t# ptr" %}
10757 
10758   ins_encode %{
10759     __ add(as_Register($dst$$reg),
10760            as_Register($src1$$reg),
10761            as_Register($src2$$reg));
10762   %}
10763 
10764   ins_pipe(ialu_reg_reg);
10765 %}
10766 
10767 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10768   match(Set dst (AddP src1 (ConvI2L src2)));
10769 
10770   ins_cost(1.9 * INSN_COST);
10771   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10772 
10773   ins_encode %{
10774     __ add(as_Register($dst$$reg),
10775            as_Register($src1$$reg),
10776            as_Register($src2$$reg), ext::sxtw);
10777   %}
10778 
10779   ins_pipe(ialu_reg_reg);
10780 %}
10781 
10782 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10783   match(Set dst (AddP src1 (LShiftL src2 scale)));
10784 
10785   ins_cost(1.9 * INSN_COST);
10786   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10787 
10788   ins_encode %{
10789     __ lea(as_Register($dst$$reg),
10790            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10791                    Address::lsl($scale$$constant)));
10792   %}
10793 
10794   ins_pipe(ialu_reg_reg_shift);
10795 %}
10796 
10797 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10798   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10799 
10800   ins_cost(1.9 * INSN_COST);
10801   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10802 
10803   ins_encode %{
10804     __ lea(as_Register($dst$$reg),
10805            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10806                    Address::sxtw($scale$$constant)));
10807   %}
10808 
10809   ins_pipe(ialu_reg_reg_shift);
10810 %}
10811 
10812 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10813   match(Set dst (LShiftL (ConvI2L src) scale));
10814 
10815   ins_cost(INSN_COST);
10816   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10817 
10818   ins_encode %{
10819     __ sbfiz(as_Register($dst$$reg),
10820           as_Register($src$$reg),
10821           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10822   %}
10823 
10824   ins_pipe(ialu_reg_shift);
10825 %}
10826 
10827 // Pointer Immediate Addition
10828 // n.b. this needs to be more expensive than using an indirect memory
10829 // operand
10830 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10831   match(Set dst (AddP src1 src2));
10832 
10833   ins_cost(INSN_COST);
10834   format %{ "add $dst, $src1, $src2\t# ptr" %}
10835 
10836   // use opcode to indicate that this is an add not a sub
10837   opcode(0x0);
10838 
10839   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10840 
10841   ins_pipe(ialu_reg_imm);
10842 %}
10843 
10844 // Long Addition
10845 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10846 
10847   match(Set dst (AddL src1 src2));
10848 
10849   ins_cost(INSN_COST);
10850   format %{ "add  $dst, $src1, $src2" %}
10851 
10852   ins_encode %{
10853     __ add(as_Register($dst$$reg),
10854            as_Register($src1$$reg),
10855            as_Register($src2$$reg));
10856   %}
10857 
10858   ins_pipe(ialu_reg_reg);
10859 %}
10860 
10861 // No constant pool entries requiredLong Immediate Addition.
10862 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10863   match(Set dst (AddL src1 src2));
10864 
10865   ins_cost(INSN_COST);
10866   format %{ "add $dst, $src1, $src2" %}
10867 
10868   // use opcode to indicate that this is an add not a sub
10869   opcode(0x0);
10870 
10871   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10872 
10873   ins_pipe(ialu_reg_imm);
10874 %}
10875 
10876 // Integer Subtraction
10877 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10878   match(Set dst (SubI src1 src2));
10879 
10880   ins_cost(INSN_COST);
10881   format %{ "subw  $dst, $src1, $src2" %}
10882 
10883   ins_encode %{
10884     __ subw(as_Register($dst$$reg),
10885             as_Register($src1$$reg),
10886             as_Register($src2$$reg));
10887   %}
10888 
10889   ins_pipe(ialu_reg_reg);
10890 %}
10891 
10892 // Immediate Subtraction
10893 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10894   match(Set dst (SubI src1 src2));
10895 
10896   ins_cost(INSN_COST);
10897   format %{ "subw $dst, $src1, $src2" %}
10898 
10899   // use opcode to indicate that this is a sub not an add
10900   opcode(0x1);
10901 
10902   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10903 
10904   ins_pipe(ialu_reg_imm);
10905 %}
10906 
10907 // Long Subtraction
10908 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10909 
10910   match(Set dst (SubL src1 src2));
10911 
10912   ins_cost(INSN_COST);
10913   format %{ "sub  $dst, $src1, $src2" %}
10914 
10915   ins_encode %{
10916     __ sub(as_Register($dst$$reg),
10917            as_Register($src1$$reg),
10918            as_Register($src2$$reg));
10919   %}
10920 
10921   ins_pipe(ialu_reg_reg);
10922 %}
10923 
10924 // No constant pool entries requiredLong Immediate Subtraction.
10925 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10926   match(Set dst (SubL src1 src2));
10927 
10928   ins_cost(INSN_COST);
10929   format %{ "sub$dst, $src1, $src2" %}
10930 
10931   // use opcode to indicate that this is a sub not an add
10932   opcode(0x1);
10933 
10934   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10935 
10936   ins_pipe(ialu_reg_imm);
10937 %}
10938 
10939 // Integer Negation (special case for sub)
10940 
10941 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10942   match(Set dst (SubI zero src));
10943 
10944   ins_cost(INSN_COST);
10945   format %{ "negw $dst, $src\t# int" %}
10946 
10947   ins_encode %{
10948     __ negw(as_Register($dst$$reg),
10949             as_Register($src$$reg));
10950   %}
10951 
10952   ins_pipe(ialu_reg);
10953 %}
10954 
10955 // Long Negation
10956 
10957 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10958   match(Set dst (SubL zero src));
10959 
10960   ins_cost(INSN_COST);
10961   format %{ "neg $dst, $src\t# long" %}
10962 
10963   ins_encode %{
10964     __ neg(as_Register($dst$$reg),
10965            as_Register($src$$reg));
10966   %}
10967 
10968   ins_pipe(ialu_reg);
10969 %}
10970 
10971 // Integer Multiply
10972 
10973 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10974   match(Set dst (MulI src1 src2));
10975 
10976   ins_cost(INSN_COST * 3);
10977   format %{ "mulw  $dst, $src1, $src2" %}
10978 
10979   ins_encode %{
10980     __ mulw(as_Register($dst$$reg),
10981             as_Register($src1$$reg),
10982             as_Register($src2$$reg));
10983   %}
10984 
10985   ins_pipe(imul_reg_reg);
10986 %}
10987 
10988 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10989   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10990 
10991   ins_cost(INSN_COST * 3);
10992   format %{ "smull  $dst, $src1, $src2" %}
10993 
10994   ins_encode %{
10995     __ smull(as_Register($dst$$reg),
10996              as_Register($src1$$reg),
10997              as_Register($src2$$reg));
10998   %}
10999 
11000   ins_pipe(imul_reg_reg);
11001 %}
11002 
11003 // Long Multiply
11004 
11005 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11006   match(Set dst (MulL src1 src2));
11007 
11008   ins_cost(INSN_COST * 5);
11009   format %{ "mul  $dst, $src1, $src2" %}
11010 
11011   ins_encode %{
11012     __ mul(as_Register($dst$$reg),
11013            as_Register($src1$$reg),
11014            as_Register($src2$$reg));
11015   %}
11016 
11017   ins_pipe(lmul_reg_reg);
11018 %}
11019 
11020 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11021 %{
11022   match(Set dst (MulHiL src1 src2));
11023 
11024   ins_cost(INSN_COST * 7);
11025   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
11026 
11027   ins_encode %{
11028     __ smulh(as_Register($dst$$reg),
11029              as_Register($src1$$reg),
11030              as_Register($src2$$reg));
11031   %}
11032 
11033   ins_pipe(lmul_reg_reg);
11034 %}
11035 
11036 // Combined Integer Multiply & Add/Sub
11037 
11038 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11039   match(Set dst (AddI src3 (MulI src1 src2)));
11040 
11041   ins_cost(INSN_COST * 3);
11042   format %{ "madd  $dst, $src1, $src2, $src3" %}
11043 
11044   ins_encode %{
11045     __ maddw(as_Register($dst$$reg),
11046              as_Register($src1$$reg),
11047              as_Register($src2$$reg),
11048              as_Register($src3$$reg));
11049   %}
11050 
11051   ins_pipe(imac_reg_reg);
11052 %}
11053 
11054 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11055   match(Set dst (SubI src3 (MulI src1 src2)));
11056 
11057   ins_cost(INSN_COST * 3);
11058   format %{ "msub  $dst, $src1, $src2, $src3" %}
11059 
11060   ins_encode %{
11061     __ msubw(as_Register($dst$$reg),
11062              as_Register($src1$$reg),
11063              as_Register($src2$$reg),
11064              as_Register($src3$$reg));
11065   %}
11066 
11067   ins_pipe(imac_reg_reg);
11068 %}
11069 
11070 // Combined Integer Multiply & Neg
11071 
11072 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11073   match(Set dst (MulI (SubI zero src1) src2));
11074 
11075   ins_cost(INSN_COST * 3);
11076   format %{ "mneg  $dst, $src1, $src2" %}
11077 
11078   ins_encode %{
11079     __ mnegw(as_Register($dst$$reg),
11080              as_Register($src1$$reg),
11081              as_Register($src2$$reg));
11082   %}
11083 
11084   ins_pipe(imac_reg_reg);
11085 %}
11086 
11087 // Combined Long Multiply & Add/Sub
11088 
11089 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11090   match(Set dst (AddL src3 (MulL src1 src2)));
11091 
11092   ins_cost(INSN_COST * 5);
11093   format %{ "madd  $dst, $src1, $src2, $src3" %}
11094 
11095   ins_encode %{
11096     __ madd(as_Register($dst$$reg),
11097             as_Register($src1$$reg),
11098             as_Register($src2$$reg),
11099             as_Register($src3$$reg));
11100   %}
11101 
11102   ins_pipe(lmac_reg_reg);
11103 %}
11104 
11105 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11106   match(Set dst (SubL src3 (MulL src1 src2)));
11107 
11108   ins_cost(INSN_COST * 5);
11109   format %{ "msub  $dst, $src1, $src2, $src3" %}
11110 
11111   ins_encode %{
11112     __ msub(as_Register($dst$$reg),
11113             as_Register($src1$$reg),
11114             as_Register($src2$$reg),
11115             as_Register($src3$$reg));
11116   %}
11117 
11118   ins_pipe(lmac_reg_reg);
11119 %}
11120 
11121 // Combined Long Multiply & Neg
11122 
11123 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11124   match(Set dst (MulL (SubL zero src1) src2));
11125 
11126   ins_cost(INSN_COST * 5);
11127   format %{ "mneg  $dst, $src1, $src2" %}
11128 
11129   ins_encode %{
11130     __ mneg(as_Register($dst$$reg),
11131             as_Register($src1$$reg),
11132             as_Register($src2$$reg));
11133   %}
11134 
11135   ins_pipe(lmac_reg_reg);
11136 %}
11137 
11138 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11139 
11140 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11141   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11142 
11143   ins_cost(INSN_COST * 3);
11144   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11145 
11146   ins_encode %{
11147     __ smaddl(as_Register($dst$$reg),
11148               as_Register($src1$$reg),
11149               as_Register($src2$$reg),
11150               as_Register($src3$$reg));
11151   %}
11152 
11153   ins_pipe(imac_reg_reg);
11154 %}
11155 
11156 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11157   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11158 
11159   ins_cost(INSN_COST * 3);
11160   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11161 
11162   ins_encode %{
11163     __ smsubl(as_Register($dst$$reg),
11164               as_Register($src1$$reg),
11165               as_Register($src2$$reg),
11166               as_Register($src3$$reg));
11167   %}
11168 
11169   ins_pipe(imac_reg_reg);
11170 %}
11171 
11172 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11173   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11174 
11175   ins_cost(INSN_COST * 3);
11176   format %{ "smnegl  $dst, $src1, $src2" %}
11177 
11178   ins_encode %{
11179     __ smnegl(as_Register($dst$$reg),
11180               as_Register($src1$$reg),
11181               as_Register($src2$$reg));
11182   %}
11183 
11184   ins_pipe(imac_reg_reg);
11185 %}
11186 
11187 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11188 
11189 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11190   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11191 
11192   ins_cost(INSN_COST * 5);
11193   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11194             "maddw $dst, $src3, $src4, rscratch1" %}
11195 
11196   ins_encode %{
11197     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11198     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11199 
11200   ins_pipe(imac_reg_reg);
11201 %}
11202 
11203 // Integer Divide
11204 
11205 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11206   match(Set dst (DivI src1 src2));
11207 
11208   ins_cost(INSN_COST * 19);
11209   format %{ "sdivw  $dst, $src1, $src2" %}
11210 
11211   ins_encode(aarch64_enc_divw(dst, src1, src2));
11212   ins_pipe(idiv_reg_reg);
11213 %}
11214 
11215 // Long Divide
11216 
11217 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11218   match(Set dst (DivL src1 src2));
11219 
11220   ins_cost(INSN_COST * 35);
11221   format %{ "sdiv   $dst, $src1, $src2" %}
11222 
11223   ins_encode(aarch64_enc_div(dst, src1, src2));
11224   ins_pipe(ldiv_reg_reg);
11225 %}
11226 
11227 // Integer Remainder
11228 
11229 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11230   match(Set dst (ModI src1 src2));
11231 
11232   ins_cost(INSN_COST * 22);
11233   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11234             "msubw($dst, rscratch1, $src2, $src1" %}
11235 
11236   ins_encode(aarch64_enc_modw(dst, src1, src2));
11237   ins_pipe(idiv_reg_reg);
11238 %}
11239 
11240 // Long Remainder
11241 
11242 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11243   match(Set dst (ModL src1 src2));
11244 
11245   ins_cost(INSN_COST * 38);
11246   format %{ "sdiv   rscratch1, $src1, $src2\n"
11247             "msub($dst, rscratch1, $src2, $src1" %}
11248 
11249   ins_encode(aarch64_enc_mod(dst, src1, src2));
11250   ins_pipe(ldiv_reg_reg);
11251 %}
11252 
11253 // Integer Shifts
11254 
11255 // Shift Left Register
11256 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11257   match(Set dst (LShiftI src1 src2));
11258 
11259   ins_cost(INSN_COST * 2);
11260   format %{ "lslvw  $dst, $src1, $src2" %}
11261 
11262   ins_encode %{
11263     __ lslvw(as_Register($dst$$reg),
11264              as_Register($src1$$reg),
11265              as_Register($src2$$reg));
11266   %}
11267 
11268   ins_pipe(ialu_reg_reg_vshift);
11269 %}
11270 
11271 // Shift Left Immediate
11272 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11273   match(Set dst (LShiftI src1 src2));
11274 
11275   ins_cost(INSN_COST);
11276   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11277 
11278   ins_encode %{
11279     __ lslw(as_Register($dst$$reg),
11280             as_Register($src1$$reg),
11281             $src2$$constant & 0x1f);
11282   %}
11283 
11284   ins_pipe(ialu_reg_shift);
11285 %}
11286 
11287 // Shift Right Logical Register
11288 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11289   match(Set dst (URShiftI src1 src2));
11290 
11291   ins_cost(INSN_COST * 2);
11292   format %{ "lsrvw  $dst, $src1, $src2" %}
11293 
11294   ins_encode %{
11295     __ lsrvw(as_Register($dst$$reg),
11296              as_Register($src1$$reg),
11297              as_Register($src2$$reg));
11298   %}
11299 
11300   ins_pipe(ialu_reg_reg_vshift);
11301 %}
11302 
11303 // Shift Right Logical Immediate
11304 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11305   match(Set dst (URShiftI src1 src2));
11306 
11307   ins_cost(INSN_COST);
11308   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11309 
11310   ins_encode %{
11311     __ lsrw(as_Register($dst$$reg),
11312             as_Register($src1$$reg),
11313             $src2$$constant & 0x1f);
11314   %}
11315 
11316   ins_pipe(ialu_reg_shift);
11317 %}
11318 
11319 // Shift Right Arithmetic Register
11320 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11321   match(Set dst (RShiftI src1 src2));
11322 
11323   ins_cost(INSN_COST * 2);
11324   format %{ "asrvw  $dst, $src1, $src2" %}
11325 
11326   ins_encode %{
11327     __ asrvw(as_Register($dst$$reg),
11328              as_Register($src1$$reg),
11329              as_Register($src2$$reg));
11330   %}
11331 
11332   ins_pipe(ialu_reg_reg_vshift);
11333 %}
11334 
11335 // Shift Right Arithmetic Immediate
11336 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11337   match(Set dst (RShiftI src1 src2));
11338 
11339   ins_cost(INSN_COST);
11340   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11341 
11342   ins_encode %{
11343     __ asrw(as_Register($dst$$reg),
11344             as_Register($src1$$reg),
11345             $src2$$constant & 0x1f);
11346   %}
11347 
11348   ins_pipe(ialu_reg_shift);
11349 %}
11350 
11351 // Combined Int Mask and Right Shift (using UBFM)
11352 // TODO
11353 
11354 // Long Shifts
11355 
11356 // Shift Left Register
11357 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11358   match(Set dst (LShiftL src1 src2));
11359 
11360   ins_cost(INSN_COST * 2);
11361   format %{ "lslv  $dst, $src1, $src2" %}
11362 
11363   ins_encode %{
11364     __ lslv(as_Register($dst$$reg),
11365             as_Register($src1$$reg),
11366             as_Register($src2$$reg));
11367   %}
11368 
11369   ins_pipe(ialu_reg_reg_vshift);
11370 %}
11371 
11372 // Shift Left Immediate
11373 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11374   match(Set dst (LShiftL src1 src2));
11375 
11376   ins_cost(INSN_COST);
11377   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11378 
11379   ins_encode %{
11380     __ lsl(as_Register($dst$$reg),
11381             as_Register($src1$$reg),
11382             $src2$$constant & 0x3f);
11383   %}
11384 
11385   ins_pipe(ialu_reg_shift);
11386 %}
11387 
11388 // Shift Right Logical Register
11389 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11390   match(Set dst (URShiftL src1 src2));
11391 
11392   ins_cost(INSN_COST * 2);
11393   format %{ "lsrv  $dst, $src1, $src2" %}
11394 
11395   ins_encode %{
11396     __ lsrv(as_Register($dst$$reg),
11397             as_Register($src1$$reg),
11398             as_Register($src2$$reg));
11399   %}
11400 
11401   ins_pipe(ialu_reg_reg_vshift);
11402 %}
11403 
11404 // Shift Right Logical Immediate
11405 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11406   match(Set dst (URShiftL src1 src2));
11407 
11408   ins_cost(INSN_COST);
11409   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11410 
11411   ins_encode %{
11412     __ lsr(as_Register($dst$$reg),
11413            as_Register($src1$$reg),
11414            $src2$$constant & 0x3f);
11415   %}
11416 
11417   ins_pipe(ialu_reg_shift);
11418 %}
11419 
11420 // A special-case pattern for card table stores.
11421 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11422   match(Set dst (URShiftL (CastP2X src1) src2));
11423 
11424   ins_cost(INSN_COST);
11425   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11426 
11427   ins_encode %{
11428     __ lsr(as_Register($dst$$reg),
11429            as_Register($src1$$reg),
11430            $src2$$constant & 0x3f);
11431   %}
11432 
11433   ins_pipe(ialu_reg_shift);
11434 %}
11435 
11436 // Shift Right Arithmetic Register
11437 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11438   match(Set dst (RShiftL src1 src2));
11439 
11440   ins_cost(INSN_COST * 2);
11441   format %{ "asrv  $dst, $src1, $src2" %}
11442 
11443   ins_encode %{
11444     __ asrv(as_Register($dst$$reg),
11445             as_Register($src1$$reg),
11446             as_Register($src2$$reg));
11447   %}
11448 
11449   ins_pipe(ialu_reg_reg_vshift);
11450 %}
11451 
11452 // Shift Right Arithmetic Immediate
11453 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11454   match(Set dst (RShiftL src1 src2));
11455 
11456   ins_cost(INSN_COST);
11457   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11458 
11459   ins_encode %{
11460     __ asr(as_Register($dst$$reg),
11461            as_Register($src1$$reg),
11462            $src2$$constant & 0x3f);
11463   %}
11464 
11465   ins_pipe(ialu_reg_shift);
11466 %}
11467 
11468 // BEGIN This section of the file is automatically generated. Do not edit --------------
11469 // This section is generated from aarch64_ad.m4
11470 
11471 
11472 // This pattern is automatically generated from aarch64_ad.m4
11473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11474 instruct regL_not_reg(iRegLNoSp dst,
11475                          iRegL src1, immL_M1 m1,
11476                          rFlagsReg cr) %{
11477   match(Set dst (XorL src1 m1));
11478   ins_cost(INSN_COST);
11479   format %{ "eon  $dst, $src1, zr" %}
11480 
11481   ins_encode %{
11482     __ eon(as_Register($dst$$reg),
11483               as_Register($src1$$reg),
11484               zr,
11485               Assembler::LSL, 0);
11486   %}
11487 
11488   ins_pipe(ialu_reg);
11489 %}
11490 
11491 // This pattern is automatically generated from aarch64_ad.m4
11492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11493 instruct regI_not_reg(iRegINoSp dst,
11494                          iRegIorL2I src1, immI_M1 m1,
11495                          rFlagsReg cr) %{
11496   match(Set dst (XorI src1 m1));
11497   ins_cost(INSN_COST);
11498   format %{ "eonw  $dst, $src1, zr" %}
11499 
11500   ins_encode %{
11501     __ eonw(as_Register($dst$$reg),
11502               as_Register($src1$$reg),
11503               zr,
11504               Assembler::LSL, 0);
11505   %}
11506 
11507   ins_pipe(ialu_reg);
11508 %}
11509 
11510 // This pattern is automatically generated from aarch64_ad.m4
11511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11512 instruct AndI_reg_not_reg(iRegINoSp dst,
11513                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11514   match(Set dst (AndI src1 (XorI src2 m1)));
11515   ins_cost(INSN_COST);
11516   format %{ "bicw  $dst, $src1, $src2" %}
11517 
11518   ins_encode %{
11519     __ bicw(as_Register($dst$$reg),
11520               as_Register($src1$$reg),
11521               as_Register($src2$$reg),
11522               Assembler::LSL, 0);
11523   %}
11524 
11525   ins_pipe(ialu_reg_reg);
11526 %}
11527 
11528 // This pattern is automatically generated from aarch64_ad.m4
11529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11530 instruct AndL_reg_not_reg(iRegLNoSp dst,
11531                          iRegL src1, iRegL src2, immL_M1 m1) %{
11532   match(Set dst (AndL src1 (XorL src2 m1)));
11533   ins_cost(INSN_COST);
11534   format %{ "bic  $dst, $src1, $src2" %}
11535 
11536   ins_encode %{
11537     __ bic(as_Register($dst$$reg),
11538               as_Register($src1$$reg),
11539               as_Register($src2$$reg),
11540               Assembler::LSL, 0);
11541   %}
11542 
11543   ins_pipe(ialu_reg_reg);
11544 %}
11545 
11546 // This pattern is automatically generated from aarch64_ad.m4
11547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11548 instruct OrI_reg_not_reg(iRegINoSp dst,
11549                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11550   match(Set dst (OrI src1 (XorI src2 m1)));
11551   ins_cost(INSN_COST);
11552   format %{ "ornw  $dst, $src1, $src2" %}
11553 
11554   ins_encode %{
11555     __ ornw(as_Register($dst$$reg),
11556               as_Register($src1$$reg),
11557               as_Register($src2$$reg),
11558               Assembler::LSL, 0);
11559   %}
11560 
11561   ins_pipe(ialu_reg_reg);
11562 %}
11563 
11564 // This pattern is automatically generated from aarch64_ad.m4
11565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11566 instruct OrL_reg_not_reg(iRegLNoSp dst,
11567                          iRegL src1, iRegL src2, immL_M1 m1) %{
11568   match(Set dst (OrL src1 (XorL src2 m1)));
11569   ins_cost(INSN_COST);
11570   format %{ "orn  $dst, $src1, $src2" %}
11571 
11572   ins_encode %{
11573     __ orn(as_Register($dst$$reg),
11574               as_Register($src1$$reg),
11575               as_Register($src2$$reg),
11576               Assembler::LSL, 0);
11577   %}
11578 
11579   ins_pipe(ialu_reg_reg);
11580 %}
11581 
11582 // This pattern is automatically generated from aarch64_ad.m4
11583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11584 instruct XorI_reg_not_reg(iRegINoSp dst,
11585                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11586   match(Set dst (XorI m1 (XorI src2 src1)));
11587   ins_cost(INSN_COST);
11588   format %{ "eonw  $dst, $src1, $src2" %}
11589 
11590   ins_encode %{
11591     __ eonw(as_Register($dst$$reg),
11592               as_Register($src1$$reg),
11593               as_Register($src2$$reg),
11594               Assembler::LSL, 0);
11595   %}
11596 
11597   ins_pipe(ialu_reg_reg);
11598 %}
11599 
11600 // This pattern is automatically generated from aarch64_ad.m4
11601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11602 instruct XorL_reg_not_reg(iRegLNoSp dst,
11603                          iRegL src1, iRegL src2, immL_M1 m1) %{
11604   match(Set dst (XorL m1 (XorL src2 src1)));
11605   ins_cost(INSN_COST);
11606   format %{ "eon  $dst, $src1, $src2" %}
11607 
11608   ins_encode %{
11609     __ eon(as_Register($dst$$reg),
11610               as_Register($src1$$reg),
11611               as_Register($src2$$reg),
11612               Assembler::LSL, 0);
11613   %}
11614 
11615   ins_pipe(ialu_reg_reg);
11616 %}
11617 
11618 // This pattern is automatically generated from aarch64_ad.m4
11619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11620 // val & (-1 ^ (val >>> shift)) ==> bicw
11621 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11622                          iRegIorL2I src1, iRegIorL2I src2,
11623                          immI src3, immI_M1 src4) %{
11624   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11625   ins_cost(1.9 * INSN_COST);
11626   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11627 
11628   ins_encode %{
11629     __ bicw(as_Register($dst$$reg),
11630               as_Register($src1$$reg),
11631               as_Register($src2$$reg),
11632               Assembler::LSR,
11633               $src3$$constant & 0x1f);
11634   %}
11635 
11636   ins_pipe(ialu_reg_reg_shift);
11637 %}
11638 
11639 // This pattern is automatically generated from aarch64_ad.m4
11640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11641 // val & (-1 ^ (val >>> shift)) ==> bic
11642 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11643                          iRegL src1, iRegL src2,
11644                          immI src3, immL_M1 src4) %{
11645   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11646   ins_cost(1.9 * INSN_COST);
11647   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11648 
11649   ins_encode %{
11650     __ bic(as_Register($dst$$reg),
11651               as_Register($src1$$reg),
11652               as_Register($src2$$reg),
11653               Assembler::LSR,
11654               $src3$$constant & 0x3f);
11655   %}
11656 
11657   ins_pipe(ialu_reg_reg_shift);
11658 %}
11659 
11660 // This pattern is automatically generated from aarch64_ad.m4
11661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11662 // val & (-1 ^ (val >> shift)) ==> bicw
11663 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11664                          iRegIorL2I src1, iRegIorL2I src2,
11665                          immI src3, immI_M1 src4) %{
11666   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11667   ins_cost(1.9 * INSN_COST);
11668   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11669 
11670   ins_encode %{
11671     __ bicw(as_Register($dst$$reg),
11672               as_Register($src1$$reg),
11673               as_Register($src2$$reg),
11674               Assembler::ASR,
11675               $src3$$constant & 0x1f);
11676   %}
11677 
11678   ins_pipe(ialu_reg_reg_shift);
11679 %}
11680 
11681 // This pattern is automatically generated from aarch64_ad.m4
11682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11683 // val & (-1 ^ (val >> shift)) ==> bic
11684 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11685                          iRegL src1, iRegL src2,
11686                          immI src3, immL_M1 src4) %{
11687   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11688   ins_cost(1.9 * INSN_COST);
11689   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11690 
11691   ins_encode %{
11692     __ bic(as_Register($dst$$reg),
11693               as_Register($src1$$reg),
11694               as_Register($src2$$reg),
11695               Assembler::ASR,
11696               $src3$$constant & 0x3f);
11697   %}
11698 
11699   ins_pipe(ialu_reg_reg_shift);
11700 %}
11701 
11702 // This pattern is automatically generated from aarch64_ad.m4
11703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11704 // val & (-1 ^ (val ror shift)) ==> bicw
11705 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11706                          iRegIorL2I src1, iRegIorL2I src2,
11707                          immI src3, immI_M1 src4) %{
11708   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11709   ins_cost(1.9 * INSN_COST);
11710   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11711 
11712   ins_encode %{
11713     __ bicw(as_Register($dst$$reg),
11714               as_Register($src1$$reg),
11715               as_Register($src2$$reg),
11716               Assembler::ROR,
11717               $src3$$constant & 0x1f);
11718   %}
11719 
11720   ins_pipe(ialu_reg_reg_shift);
11721 %}
11722 
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 // val & (-1 ^ (val ror shift)) ==> bic
11726 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11727                          iRegL src1, iRegL src2,
11728                          immI src3, immL_M1 src4) %{
11729   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11730   ins_cost(1.9 * INSN_COST);
11731   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11732 
11733   ins_encode %{
11734     __ bic(as_Register($dst$$reg),
11735               as_Register($src1$$reg),
11736               as_Register($src2$$reg),
11737               Assembler::ROR,
11738               $src3$$constant & 0x3f);
11739   %}
11740 
11741   ins_pipe(ialu_reg_reg_shift);
11742 %}
11743 
11744 // This pattern is automatically generated from aarch64_ad.m4
11745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11746 // val & (-1 ^ (val << shift)) ==> bicw
11747 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11748                          iRegIorL2I src1, iRegIorL2I src2,
11749                          immI src3, immI_M1 src4) %{
11750   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11751   ins_cost(1.9 * INSN_COST);
11752   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11753 
11754   ins_encode %{
11755     __ bicw(as_Register($dst$$reg),
11756               as_Register($src1$$reg),
11757               as_Register($src2$$reg),
11758               Assembler::LSL,
11759               $src3$$constant & 0x1f);
11760   %}
11761 
11762   ins_pipe(ialu_reg_reg_shift);
11763 %}
11764 
11765 // This pattern is automatically generated from aarch64_ad.m4
11766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11767 // val & (-1 ^ (val << shift)) ==> bic
11768 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11769                          iRegL src1, iRegL src2,
11770                          immI src3, immL_M1 src4) %{
11771   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11772   ins_cost(1.9 * INSN_COST);
11773   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11774 
11775   ins_encode %{
11776     __ bic(as_Register($dst$$reg),
11777               as_Register($src1$$reg),
11778               as_Register($src2$$reg),
11779               Assembler::LSL,
11780               $src3$$constant & 0x3f);
11781   %}
11782 
11783   ins_pipe(ialu_reg_reg_shift);
11784 %}
11785 
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11789 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11790                          iRegIorL2I src1, iRegIorL2I src2,
11791                          immI src3, immI_M1 src4) %{
11792   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11793   ins_cost(1.9 * INSN_COST);
11794   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11795 
11796   ins_encode %{
11797     __ eonw(as_Register($dst$$reg),
11798               as_Register($src1$$reg),
11799               as_Register($src2$$reg),
11800               Assembler::LSR,
11801               $src3$$constant & 0x1f);
11802   %}
11803 
11804   ins_pipe(ialu_reg_reg_shift);
11805 %}
11806 
11807 // This pattern is automatically generated from aarch64_ad.m4
11808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11809 // val ^ (-1 ^ (val >>> shift)) ==> eon
11810 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11811                          iRegL src1, iRegL src2,
11812                          immI src3, immL_M1 src4) %{
11813   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11814   ins_cost(1.9 * INSN_COST);
11815   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11816 
11817   ins_encode %{
11818     __ eon(as_Register($dst$$reg),
11819               as_Register($src1$$reg),
11820               as_Register($src2$$reg),
11821               Assembler::LSR,
11822               $src3$$constant & 0x3f);
11823   %}
11824 
11825   ins_pipe(ialu_reg_reg_shift);
11826 %}
11827 
11828 // This pattern is automatically generated from aarch64_ad.m4
11829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11830 // val ^ (-1 ^ (val >> shift)) ==> eonw
11831 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11832                          iRegIorL2I src1, iRegIorL2I src2,
11833                          immI src3, immI_M1 src4) %{
11834   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11835   ins_cost(1.9 * INSN_COST);
11836   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11837 
11838   ins_encode %{
11839     __ eonw(as_Register($dst$$reg),
11840               as_Register($src1$$reg),
11841               as_Register($src2$$reg),
11842               Assembler::ASR,
11843               $src3$$constant & 0x1f);
11844   %}
11845 
11846   ins_pipe(ialu_reg_reg_shift);
11847 %}
11848 
11849 // This pattern is automatically generated from aarch64_ad.m4
11850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11851 // val ^ (-1 ^ (val >> shift)) ==> eon
11852 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11853                          iRegL src1, iRegL src2,
11854                          immI src3, immL_M1 src4) %{
11855   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11856   ins_cost(1.9 * INSN_COST);
11857   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11858 
11859   ins_encode %{
11860     __ eon(as_Register($dst$$reg),
11861               as_Register($src1$$reg),
11862               as_Register($src2$$reg),
11863               Assembler::ASR,
11864               $src3$$constant & 0x3f);
11865   %}
11866 
11867   ins_pipe(ialu_reg_reg_shift);
11868 %}
11869 
11870 // This pattern is automatically generated from aarch64_ad.m4
11871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11872 // val ^ (-1 ^ (val ror shift)) ==> eonw
11873 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11874                          iRegIorL2I src1, iRegIorL2I src2,
11875                          immI src3, immI_M1 src4) %{
11876   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11877   ins_cost(1.9 * INSN_COST);
11878   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11879 
11880   ins_encode %{
11881     __ eonw(as_Register($dst$$reg),
11882               as_Register($src1$$reg),
11883               as_Register($src2$$reg),
11884               Assembler::ROR,
11885               $src3$$constant & 0x1f);
11886   %}
11887 
11888   ins_pipe(ialu_reg_reg_shift);
11889 %}
11890 
11891 // This pattern is automatically generated from aarch64_ad.m4
11892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11893 // val ^ (-1 ^ (val ror shift)) ==> eon
11894 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11895                          iRegL src1, iRegL src2,
11896                          immI src3, immL_M1 src4) %{
11897   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11898   ins_cost(1.9 * INSN_COST);
11899   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11900 
11901   ins_encode %{
11902     __ eon(as_Register($dst$$reg),
11903               as_Register($src1$$reg),
11904               as_Register($src2$$reg),
11905               Assembler::ROR,
11906               $src3$$constant & 0x3f);
11907   %}
11908 
11909   ins_pipe(ialu_reg_reg_shift);
11910 %}
11911 
11912 // This pattern is automatically generated from aarch64_ad.m4
11913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11914 // val ^ (-1 ^ (val << shift)) ==> eonw
11915 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11916                          iRegIorL2I src1, iRegIorL2I src2,
11917                          immI src3, immI_M1 src4) %{
11918   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11919   ins_cost(1.9 * INSN_COST);
11920   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11921 
11922   ins_encode %{
11923     __ eonw(as_Register($dst$$reg),
11924               as_Register($src1$$reg),
11925               as_Register($src2$$reg),
11926               Assembler::LSL,
11927               $src3$$constant & 0x1f);
11928   %}
11929 
11930   ins_pipe(ialu_reg_reg_shift);
11931 %}
11932 
11933 // This pattern is automatically generated from aarch64_ad.m4
11934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11935 // val ^ (-1 ^ (val << shift)) ==> eon
11936 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11937                          iRegL src1, iRegL src2,
11938                          immI src3, immL_M1 src4) %{
11939   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11940   ins_cost(1.9 * INSN_COST);
11941   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11942 
11943   ins_encode %{
11944     __ eon(as_Register($dst$$reg),
11945               as_Register($src1$$reg),
11946               as_Register($src2$$reg),
11947               Assembler::LSL,
11948               $src3$$constant & 0x3f);
11949   %}
11950 
11951   ins_pipe(ialu_reg_reg_shift);
11952 %}
11953 
11954 // This pattern is automatically generated from aarch64_ad.m4
11955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11956 // val | (-1 ^ (val >>> shift)) ==> ornw
11957 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11958                          iRegIorL2I src1, iRegIorL2I src2,
11959                          immI src3, immI_M1 src4) %{
11960   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11961   ins_cost(1.9 * INSN_COST);
11962   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11963 
11964   ins_encode %{
11965     __ ornw(as_Register($dst$$reg),
11966               as_Register($src1$$reg),
11967               as_Register($src2$$reg),
11968               Assembler::LSR,
11969               $src3$$constant & 0x1f);
11970   %}
11971 
11972   ins_pipe(ialu_reg_reg_shift);
11973 %}
11974 
11975 // This pattern is automatically generated from aarch64_ad.m4
11976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11977 // val | (-1 ^ (val >>> shift)) ==> orn
11978 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11979                          iRegL src1, iRegL src2,
11980                          immI src3, immL_M1 src4) %{
11981   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11982   ins_cost(1.9 * INSN_COST);
11983   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11984 
11985   ins_encode %{
11986     __ orn(as_Register($dst$$reg),
11987               as_Register($src1$$reg),
11988               as_Register($src2$$reg),
11989               Assembler::LSR,
11990               $src3$$constant & 0x3f);
11991   %}
11992 
11993   ins_pipe(ialu_reg_reg_shift);
11994 %}
11995 
11996 // This pattern is automatically generated from aarch64_ad.m4
11997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11998 // val | (-1 ^ (val >> shift)) ==> ornw
11999 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12000                          iRegIorL2I src1, iRegIorL2I src2,
12001                          immI src3, immI_M1 src4) %{
12002   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12003   ins_cost(1.9 * INSN_COST);
12004   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12005 
12006   ins_encode %{
12007     __ ornw(as_Register($dst$$reg),
12008               as_Register($src1$$reg),
12009               as_Register($src2$$reg),
12010               Assembler::ASR,
12011               $src3$$constant & 0x1f);
12012   %}
12013 
12014   ins_pipe(ialu_reg_reg_shift);
12015 %}
12016 
12017 // This pattern is automatically generated from aarch64_ad.m4
12018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12019 // val | (-1 ^ (val >> shift)) ==> orn
12020 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12021                          iRegL src1, iRegL src2,
12022                          immI src3, immL_M1 src4) %{
12023   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12024   ins_cost(1.9 * INSN_COST);
12025   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12026 
12027   ins_encode %{
12028     __ orn(as_Register($dst$$reg),
12029               as_Register($src1$$reg),
12030               as_Register($src2$$reg),
12031               Assembler::ASR,
12032               $src3$$constant & 0x3f);
12033   %}
12034 
12035   ins_pipe(ialu_reg_reg_shift);
12036 %}
12037 
12038 // This pattern is automatically generated from aarch64_ad.m4
12039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12040 // val | (-1 ^ (val ror shift)) ==> ornw
12041 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12042                          iRegIorL2I src1, iRegIorL2I src2,
12043                          immI src3, immI_M1 src4) %{
12044   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12045   ins_cost(1.9 * INSN_COST);
12046   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12047 
12048   ins_encode %{
12049     __ ornw(as_Register($dst$$reg),
12050               as_Register($src1$$reg),
12051               as_Register($src2$$reg),
12052               Assembler::ROR,
12053               $src3$$constant & 0x1f);
12054   %}
12055 
12056   ins_pipe(ialu_reg_reg_shift);
12057 %}
12058 
12059 // This pattern is automatically generated from aarch64_ad.m4
12060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12061 // val | (-1 ^ (val ror shift)) ==> orn
12062 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12063                          iRegL src1, iRegL src2,
12064                          immI src3, immL_M1 src4) %{
12065   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12066   ins_cost(1.9 * INSN_COST);
12067   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12068 
12069   ins_encode %{
12070     __ orn(as_Register($dst$$reg),
12071               as_Register($src1$$reg),
12072               as_Register($src2$$reg),
12073               Assembler::ROR,
12074               $src3$$constant & 0x3f);
12075   %}
12076 
12077   ins_pipe(ialu_reg_reg_shift);
12078 %}
12079 
12080 // This pattern is automatically generated from aarch64_ad.m4
12081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12082 // val | (-1 ^ (val << shift)) ==> ornw
12083 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12084                          iRegIorL2I src1, iRegIorL2I src2,
12085                          immI src3, immI_M1 src4) %{
12086   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12087   ins_cost(1.9 * INSN_COST);
12088   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12089 
12090   ins_encode %{
12091     __ ornw(as_Register($dst$$reg),
12092               as_Register($src1$$reg),
12093               as_Register($src2$$reg),
12094               Assembler::LSL,
12095               $src3$$constant & 0x1f);
12096   %}
12097 
12098   ins_pipe(ialu_reg_reg_shift);
12099 %}
12100 
12101 // This pattern is automatically generated from aarch64_ad.m4
12102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12103 // val | (-1 ^ (val << shift)) ==> orn
12104 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12105                          iRegL src1, iRegL src2,
12106                          immI src3, immL_M1 src4) %{
12107   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12108   ins_cost(1.9 * INSN_COST);
12109   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12110 
12111   ins_encode %{
12112     __ orn(as_Register($dst$$reg),
12113               as_Register($src1$$reg),
12114               as_Register($src2$$reg),
12115               Assembler::LSL,
12116               $src3$$constant & 0x3f);
12117   %}
12118 
12119   ins_pipe(ialu_reg_reg_shift);
12120 %}
12121 
12122 // This pattern is automatically generated from aarch64_ad.m4
12123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12124 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12125                          iRegIorL2I src1, iRegIorL2I src2,
12126                          immI src3) %{
12127   match(Set dst (AndI src1 (URShiftI src2 src3)));
12128 
12129   ins_cost(1.9 * INSN_COST);
12130   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12131 
12132   ins_encode %{
12133     __ andw(as_Register($dst$$reg),
12134               as_Register($src1$$reg),
12135               as_Register($src2$$reg),
12136               Assembler::LSR,
12137               $src3$$constant & 0x1f);
12138   %}
12139 
12140   ins_pipe(ialu_reg_reg_shift);
12141 %}
12142 
12143 // This pattern is automatically generated from aarch64_ad.m4
12144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12145 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12146                          iRegL src1, iRegL src2,
12147                          immI src3) %{
12148   match(Set dst (AndL src1 (URShiftL src2 src3)));
12149 
12150   ins_cost(1.9 * INSN_COST);
12151   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12152 
12153   ins_encode %{
12154     __ andr(as_Register($dst$$reg),
12155               as_Register($src1$$reg),
12156               as_Register($src2$$reg),
12157               Assembler::LSR,
12158               $src3$$constant & 0x3f);
12159   %}
12160 
12161   ins_pipe(ialu_reg_reg_shift);
12162 %}
12163 
12164 // This pattern is automatically generated from aarch64_ad.m4
12165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12166 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12167                          iRegIorL2I src1, iRegIorL2I src2,
12168                          immI src3) %{
12169   match(Set dst (AndI src1 (RShiftI src2 src3)));
12170 
12171   ins_cost(1.9 * INSN_COST);
12172   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12173 
12174   ins_encode %{
12175     __ andw(as_Register($dst$$reg),
12176               as_Register($src1$$reg),
12177               as_Register($src2$$reg),
12178               Assembler::ASR,
12179               $src3$$constant & 0x1f);
12180   %}
12181 
12182   ins_pipe(ialu_reg_reg_shift);
12183 %}
12184 
12185 // This pattern is automatically generated from aarch64_ad.m4
12186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12187 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12188                          iRegL src1, iRegL src2,
12189                          immI src3) %{
12190   match(Set dst (AndL src1 (RShiftL src2 src3)));
12191 
12192   ins_cost(1.9 * INSN_COST);
12193   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12194 
12195   ins_encode %{
12196     __ andr(as_Register($dst$$reg),
12197               as_Register($src1$$reg),
12198               as_Register($src2$$reg),
12199               Assembler::ASR,
12200               $src3$$constant & 0x3f);
12201   %}
12202 
12203   ins_pipe(ialu_reg_reg_shift);
12204 %}
12205 
12206 // This pattern is automatically generated from aarch64_ad.m4
12207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12208 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12209                          iRegIorL2I src1, iRegIorL2I src2,
12210                          immI src3) %{
12211   match(Set dst (AndI src1 (LShiftI src2 src3)));
12212 
12213   ins_cost(1.9 * INSN_COST);
12214   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12215 
12216   ins_encode %{
12217     __ andw(as_Register($dst$$reg),
12218               as_Register($src1$$reg),
12219               as_Register($src2$$reg),
12220               Assembler::LSL,
12221               $src3$$constant & 0x1f);
12222   %}
12223 
12224   ins_pipe(ialu_reg_reg_shift);
12225 %}
12226 
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12230                          iRegL src1, iRegL src2,
12231                          immI src3) %{
12232   match(Set dst (AndL src1 (LShiftL src2 src3)));
12233 
12234   ins_cost(1.9 * INSN_COST);
12235   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12236 
12237   ins_encode %{
12238     __ andr(as_Register($dst$$reg),
12239               as_Register($src1$$reg),
12240               as_Register($src2$$reg),
12241               Assembler::LSL,
12242               $src3$$constant & 0x3f);
12243   %}
12244 
12245   ins_pipe(ialu_reg_reg_shift);
12246 %}
12247 
12248 // This pattern is automatically generated from aarch64_ad.m4
12249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12250 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12251                          iRegIorL2I src1, iRegIorL2I src2,
12252                          immI src3) %{
12253   match(Set dst (AndI src1 (RotateRight src2 src3)));
12254 
12255   ins_cost(1.9 * INSN_COST);
12256   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12257 
12258   ins_encode %{
12259     __ andw(as_Register($dst$$reg),
12260               as_Register($src1$$reg),
12261               as_Register($src2$$reg),
12262               Assembler::ROR,
12263               $src3$$constant & 0x1f);
12264   %}
12265 
12266   ins_pipe(ialu_reg_reg_shift);
12267 %}
12268 
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12272                          iRegL src1, iRegL src2,
12273                          immI src3) %{
12274   match(Set dst (AndL src1 (RotateRight src2 src3)));
12275 
12276   ins_cost(1.9 * INSN_COST);
12277   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12278 
12279   ins_encode %{
12280     __ andr(as_Register($dst$$reg),
12281               as_Register($src1$$reg),
12282               as_Register($src2$$reg),
12283               Assembler::ROR,
12284               $src3$$constant & 0x3f);
12285   %}
12286 
12287   ins_pipe(ialu_reg_reg_shift);
12288 %}
12289 
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12293                          iRegIorL2I src1, iRegIorL2I src2,
12294                          immI src3) %{
12295   match(Set dst (XorI src1 (URShiftI src2 src3)));
12296 
12297   ins_cost(1.9 * INSN_COST);
12298   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12299 
12300   ins_encode %{
12301     __ eorw(as_Register($dst$$reg),
12302               as_Register($src1$$reg),
12303               as_Register($src2$$reg),
12304               Assembler::LSR,
12305               $src3$$constant & 0x1f);
12306   %}
12307 
12308   ins_pipe(ialu_reg_reg_shift);
12309 %}
12310 
12311 // This pattern is automatically generated from aarch64_ad.m4
12312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12313 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12314                          iRegL src1, iRegL src2,
12315                          immI src3) %{
12316   match(Set dst (XorL src1 (URShiftL src2 src3)));
12317 
12318   ins_cost(1.9 * INSN_COST);
12319   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12320 
12321   ins_encode %{
12322     __ eor(as_Register($dst$$reg),
12323               as_Register($src1$$reg),
12324               as_Register($src2$$reg),
12325               Assembler::LSR,
12326               $src3$$constant & 0x3f);
12327   %}
12328 
12329   ins_pipe(ialu_reg_reg_shift);
12330 %}
12331 
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12335                          iRegIorL2I src1, iRegIorL2I src2,
12336                          immI src3) %{
12337   match(Set dst (XorI src1 (RShiftI src2 src3)));
12338 
12339   ins_cost(1.9 * INSN_COST);
12340   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12341 
12342   ins_encode %{
12343     __ eorw(as_Register($dst$$reg),
12344               as_Register($src1$$reg),
12345               as_Register($src2$$reg),
12346               Assembler::ASR,
12347               $src3$$constant & 0x1f);
12348   %}
12349 
12350   ins_pipe(ialu_reg_reg_shift);
12351 %}
12352 
12353 // This pattern is automatically generated from aarch64_ad.m4
12354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12355 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12356                          iRegL src1, iRegL src2,
12357                          immI src3) %{
12358   match(Set dst (XorL src1 (RShiftL src2 src3)));
12359 
12360   ins_cost(1.9 * INSN_COST);
12361   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12362 
12363   ins_encode %{
12364     __ eor(as_Register($dst$$reg),
12365               as_Register($src1$$reg),
12366               as_Register($src2$$reg),
12367               Assembler::ASR,
12368               $src3$$constant & 0x3f);
12369   %}
12370 
12371   ins_pipe(ialu_reg_reg_shift);
12372 %}
12373 
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12377                          iRegIorL2I src1, iRegIorL2I src2,
12378                          immI src3) %{
12379   match(Set dst (XorI src1 (LShiftI src2 src3)));
12380 
12381   ins_cost(1.9 * INSN_COST);
12382   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12383 
12384   ins_encode %{
12385     __ eorw(as_Register($dst$$reg),
12386               as_Register($src1$$reg),
12387               as_Register($src2$$reg),
12388               Assembler::LSL,
12389               $src3$$constant & 0x1f);
12390   %}
12391 
12392   ins_pipe(ialu_reg_reg_shift);
12393 %}
12394 
12395 // This pattern is automatically generated from aarch64_ad.m4
12396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12397 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12398                          iRegL src1, iRegL src2,
12399                          immI src3) %{
12400   match(Set dst (XorL src1 (LShiftL src2 src3)));
12401 
12402   ins_cost(1.9 * INSN_COST);
12403   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12404 
12405   ins_encode %{
12406     __ eor(as_Register($dst$$reg),
12407               as_Register($src1$$reg),
12408               as_Register($src2$$reg),
12409               Assembler::LSL,
12410               $src3$$constant & 0x3f);
12411   %}
12412 
12413   ins_pipe(ialu_reg_reg_shift);
12414 %}
12415 
12416 // This pattern is automatically generated from aarch64_ad.m4
12417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12418 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12419                          iRegIorL2I src1, iRegIorL2I src2,
12420                          immI src3) %{
12421   match(Set dst (XorI src1 (RotateRight src2 src3)));
12422 
12423   ins_cost(1.9 * INSN_COST);
12424   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12425 
12426   ins_encode %{
12427     __ eorw(as_Register($dst$$reg),
12428               as_Register($src1$$reg),
12429               as_Register($src2$$reg),
12430               Assembler::ROR,
12431               $src3$$constant & 0x1f);
12432   %}
12433 
12434   ins_pipe(ialu_reg_reg_shift);
12435 %}
12436 
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12440                          iRegL src1, iRegL src2,
12441                          immI src3) %{
12442   match(Set dst (XorL src1 (RotateRight src2 src3)));
12443 
12444   ins_cost(1.9 * INSN_COST);
12445   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12446 
12447   ins_encode %{
12448     __ eor(as_Register($dst$$reg),
12449               as_Register($src1$$reg),
12450               as_Register($src2$$reg),
12451               Assembler::ROR,
12452               $src3$$constant & 0x3f);
12453   %}
12454 
12455   ins_pipe(ialu_reg_reg_shift);
12456 %}
12457 
12458 // This pattern is automatically generated from aarch64_ad.m4
12459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12460 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12461                          iRegIorL2I src1, iRegIorL2I src2,
12462                          immI src3) %{
12463   match(Set dst (OrI src1 (URShiftI src2 src3)));
12464 
12465   ins_cost(1.9 * INSN_COST);
12466   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12467 
12468   ins_encode %{
12469     __ orrw(as_Register($dst$$reg),
12470               as_Register($src1$$reg),
12471               as_Register($src2$$reg),
12472               Assembler::LSR,
12473               $src3$$constant & 0x1f);
12474   %}
12475 
12476   ins_pipe(ialu_reg_reg_shift);
12477 %}
12478 
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12482                          iRegL src1, iRegL src2,
12483                          immI src3) %{
12484   match(Set dst (OrL src1 (URShiftL src2 src3)));
12485 
12486   ins_cost(1.9 * INSN_COST);
12487   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12488 
12489   ins_encode %{
12490     __ orr(as_Register($dst$$reg),
12491               as_Register($src1$$reg),
12492               as_Register($src2$$reg),
12493               Assembler::LSR,
12494               $src3$$constant & 0x3f);
12495   %}
12496 
12497   ins_pipe(ialu_reg_reg_shift);
12498 %}
12499 
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12503                          iRegIorL2I src1, iRegIorL2I src2,
12504                          immI src3) %{
12505   match(Set dst (OrI src1 (RShiftI src2 src3)));
12506 
12507   ins_cost(1.9 * INSN_COST);
12508   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12509 
12510   ins_encode %{
12511     __ orrw(as_Register($dst$$reg),
12512               as_Register($src1$$reg),
12513               as_Register($src2$$reg),
12514               Assembler::ASR,
12515               $src3$$constant & 0x1f);
12516   %}
12517 
12518   ins_pipe(ialu_reg_reg_shift);
12519 %}
12520 
12521 // This pattern is automatically generated from aarch64_ad.m4
12522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12523 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12524                          iRegL src1, iRegL src2,
12525                          immI src3) %{
12526   match(Set dst (OrL src1 (RShiftL src2 src3)));
12527 
12528   ins_cost(1.9 * INSN_COST);
12529   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12530 
12531   ins_encode %{
12532     __ orr(as_Register($dst$$reg),
12533               as_Register($src1$$reg),
12534               as_Register($src2$$reg),
12535               Assembler::ASR,
12536               $src3$$constant & 0x3f);
12537   %}
12538 
12539   ins_pipe(ialu_reg_reg_shift);
12540 %}
12541 
12542 // This pattern is automatically generated from aarch64_ad.m4
12543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12544 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12545                          iRegIorL2I src1, iRegIorL2I src2,
12546                          immI src3) %{
12547   match(Set dst (OrI src1 (LShiftI src2 src3)));
12548 
12549   ins_cost(1.9 * INSN_COST);
12550   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12551 
12552   ins_encode %{
12553     __ orrw(as_Register($dst$$reg),
12554               as_Register($src1$$reg),
12555               as_Register($src2$$reg),
12556               Assembler::LSL,
12557               $src3$$constant & 0x1f);
12558   %}
12559 
12560   ins_pipe(ialu_reg_reg_shift);
12561 %}
12562 
12563 // This pattern is automatically generated from aarch64_ad.m4
12564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12565 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12566                          iRegL src1, iRegL src2,
12567                          immI src3) %{
12568   match(Set dst (OrL src1 (LShiftL src2 src3)));
12569 
12570   ins_cost(1.9 * INSN_COST);
12571   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12572 
12573   ins_encode %{
12574     __ orr(as_Register($dst$$reg),
12575               as_Register($src1$$reg),
12576               as_Register($src2$$reg),
12577               Assembler::LSL,
12578               $src3$$constant & 0x3f);
12579   %}
12580 
12581   ins_pipe(ialu_reg_reg_shift);
12582 %}
12583 
12584 // This pattern is automatically generated from aarch64_ad.m4
12585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12586 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12587                          iRegIorL2I src1, iRegIorL2I src2,
12588                          immI src3) %{
12589   match(Set dst (OrI src1 (RotateRight src2 src3)));
12590 
12591   ins_cost(1.9 * INSN_COST);
12592   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12593 
12594   ins_encode %{
12595     __ orrw(as_Register($dst$$reg),
12596               as_Register($src1$$reg),
12597               as_Register($src2$$reg),
12598               Assembler::ROR,
12599               $src3$$constant & 0x1f);
12600   %}
12601 
12602   ins_pipe(ialu_reg_reg_shift);
12603 %}
12604 
12605 // This pattern is automatically generated from aarch64_ad.m4
12606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12607 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12608                          iRegL src1, iRegL src2,
12609                          immI src3) %{
12610   match(Set dst (OrL src1 (RotateRight src2 src3)));
12611 
12612   ins_cost(1.9 * INSN_COST);
12613   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12614 
12615   ins_encode %{
12616     __ orr(as_Register($dst$$reg),
12617               as_Register($src1$$reg),
12618               as_Register($src2$$reg),
12619               Assembler::ROR,
12620               $src3$$constant & 0x3f);
12621   %}
12622 
12623   ins_pipe(ialu_reg_reg_shift);
12624 %}
12625 
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12629                          iRegIorL2I src1, iRegIorL2I src2,
12630                          immI src3) %{
12631   match(Set dst (AddI src1 (URShiftI src2 src3)));
12632 
12633   ins_cost(1.9 * INSN_COST);
12634   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12635 
12636   ins_encode %{
12637     __ addw(as_Register($dst$$reg),
12638               as_Register($src1$$reg),
12639               as_Register($src2$$reg),
12640               Assembler::LSR,
12641               $src3$$constant & 0x1f);
12642   %}
12643 
12644   ins_pipe(ialu_reg_reg_shift);
12645 %}
12646 
12647 // This pattern is automatically generated from aarch64_ad.m4
12648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12649 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12650                          iRegL src1, iRegL src2,
12651                          immI src3) %{
12652   match(Set dst (AddL src1 (URShiftL src2 src3)));
12653 
12654   ins_cost(1.9 * INSN_COST);
12655   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12656 
12657   ins_encode %{
12658     __ add(as_Register($dst$$reg),
12659               as_Register($src1$$reg),
12660               as_Register($src2$$reg),
12661               Assembler::LSR,
12662               $src3$$constant & 0x3f);
12663   %}
12664 
12665   ins_pipe(ialu_reg_reg_shift);
12666 %}
12667 
12668 // This pattern is automatically generated from aarch64_ad.m4
12669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12670 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12671                          iRegIorL2I src1, iRegIorL2I src2,
12672                          immI src3) %{
12673   match(Set dst (AddI src1 (RShiftI src2 src3)));
12674 
12675   ins_cost(1.9 * INSN_COST);
12676   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12677 
12678   ins_encode %{
12679     __ addw(as_Register($dst$$reg),
12680               as_Register($src1$$reg),
12681               as_Register($src2$$reg),
12682               Assembler::ASR,
12683               $src3$$constant & 0x1f);
12684   %}
12685 
12686   ins_pipe(ialu_reg_reg_shift);
12687 %}
12688 
12689 // This pattern is automatically generated from aarch64_ad.m4
12690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12691 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12692                          iRegL src1, iRegL src2,
12693                          immI src3) %{
12694   match(Set dst (AddL src1 (RShiftL src2 src3)));
12695 
12696   ins_cost(1.9 * INSN_COST);
12697   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12698 
12699   ins_encode %{
12700     __ add(as_Register($dst$$reg),
12701               as_Register($src1$$reg),
12702               as_Register($src2$$reg),
12703               Assembler::ASR,
12704               $src3$$constant & 0x3f);
12705   %}
12706 
12707   ins_pipe(ialu_reg_reg_shift);
12708 %}
12709 
12710 // This pattern is automatically generated from aarch64_ad.m4
12711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12712 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12713                          iRegIorL2I src1, iRegIorL2I src2,
12714                          immI src3) %{
12715   match(Set dst (AddI src1 (LShiftI src2 src3)));
12716 
12717   ins_cost(1.9 * INSN_COST);
12718   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12719 
12720   ins_encode %{
12721     __ addw(as_Register($dst$$reg),
12722               as_Register($src1$$reg),
12723               as_Register($src2$$reg),
12724               Assembler::LSL,
12725               $src3$$constant & 0x1f);
12726   %}
12727 
12728   ins_pipe(ialu_reg_reg_shift);
12729 %}
12730 
12731 // This pattern is automatically generated from aarch64_ad.m4
12732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12733 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12734                          iRegL src1, iRegL src2,
12735                          immI src3) %{
12736   match(Set dst (AddL src1 (LShiftL src2 src3)));
12737 
12738   ins_cost(1.9 * INSN_COST);
12739   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12740 
12741   ins_encode %{
12742     __ add(as_Register($dst$$reg),
12743               as_Register($src1$$reg),
12744               as_Register($src2$$reg),
12745               Assembler::LSL,
12746               $src3$$constant & 0x3f);
12747   %}
12748 
12749   ins_pipe(ialu_reg_reg_shift);
12750 %}
12751 
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12755                          iRegIorL2I src1, iRegIorL2I src2,
12756                          immI src3) %{
12757   match(Set dst (SubI src1 (URShiftI src2 src3)));
12758 
12759   ins_cost(1.9 * INSN_COST);
12760   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12761 
12762   ins_encode %{
12763     __ subw(as_Register($dst$$reg),
12764               as_Register($src1$$reg),
12765               as_Register($src2$$reg),
12766               Assembler::LSR,
12767               $src3$$constant & 0x1f);
12768   %}
12769 
12770   ins_pipe(ialu_reg_reg_shift);
12771 %}
12772 
12773 // This pattern is automatically generated from aarch64_ad.m4
12774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12775 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12776                          iRegL src1, iRegL src2,
12777                          immI src3) %{
12778   match(Set dst (SubL src1 (URShiftL src2 src3)));
12779 
12780   ins_cost(1.9 * INSN_COST);
12781   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12782 
12783   ins_encode %{
12784     __ sub(as_Register($dst$$reg),
12785               as_Register($src1$$reg),
12786               as_Register($src2$$reg),
12787               Assembler::LSR,
12788               $src3$$constant & 0x3f);
12789   %}
12790 
12791   ins_pipe(ialu_reg_reg_shift);
12792 %}
12793 
12794 // This pattern is automatically generated from aarch64_ad.m4
12795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12796 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12797                          iRegIorL2I src1, iRegIorL2I src2,
12798                          immI src3) %{
12799   match(Set dst (SubI src1 (RShiftI src2 src3)));
12800 
12801   ins_cost(1.9 * INSN_COST);
12802   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12803 
12804   ins_encode %{
12805     __ subw(as_Register($dst$$reg),
12806               as_Register($src1$$reg),
12807               as_Register($src2$$reg),
12808               Assembler::ASR,
12809               $src3$$constant & 0x1f);
12810   %}
12811 
12812   ins_pipe(ialu_reg_reg_shift);
12813 %}
12814 
12815 // This pattern is automatically generated from aarch64_ad.m4
12816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12817 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12818                          iRegL src1, iRegL src2,
12819                          immI src3) %{
12820   match(Set dst (SubL src1 (RShiftL src2 src3)));
12821 
12822   ins_cost(1.9 * INSN_COST);
12823   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12824 
12825   ins_encode %{
12826     __ sub(as_Register($dst$$reg),
12827               as_Register($src1$$reg),
12828               as_Register($src2$$reg),
12829               Assembler::ASR,
12830               $src3$$constant & 0x3f);
12831   %}
12832 
12833   ins_pipe(ialu_reg_reg_shift);
12834 %}
12835 
12836 // This pattern is automatically generated from aarch64_ad.m4
12837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12838 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12839                          iRegIorL2I src1, iRegIorL2I src2,
12840                          immI src3) %{
12841   match(Set dst (SubI src1 (LShiftI src2 src3)));
12842 
12843   ins_cost(1.9 * INSN_COST);
12844   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12845 
12846   ins_encode %{
12847     __ subw(as_Register($dst$$reg),
12848               as_Register($src1$$reg),
12849               as_Register($src2$$reg),
12850               Assembler::LSL,
12851               $src3$$constant & 0x1f);
12852   %}
12853 
12854   ins_pipe(ialu_reg_reg_shift);
12855 %}
12856 
12857 // This pattern is automatically generated from aarch64_ad.m4
12858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12859 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12860                          iRegL src1, iRegL src2,
12861                          immI src3) %{
12862   match(Set dst (SubL src1 (LShiftL src2 src3)));
12863 
12864   ins_cost(1.9 * INSN_COST);
12865   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12866 
12867   ins_encode %{
12868     __ sub(as_Register($dst$$reg),
12869               as_Register($src1$$reg),
12870               as_Register($src2$$reg),
12871               Assembler::LSL,
12872               $src3$$constant & 0x3f);
12873   %}
12874 
12875   ins_pipe(ialu_reg_reg_shift);
12876 %}
12877 
12878 // This pattern is automatically generated from aarch64_ad.m4
12879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12880 
12881 // Shift Left followed by Shift Right.
12882 // This idiom is used by the compiler for the i2b bytecode etc.
12883 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12884 %{
12885   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12886   ins_cost(INSN_COST * 2);
12887   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12888   ins_encode %{
12889     int lshift = $lshift_count$$constant & 63;
12890     int rshift = $rshift_count$$constant & 63;
12891     int s = 63 - lshift;
12892     int r = (rshift - lshift) & 63;
12893     __ sbfm(as_Register($dst$$reg),
12894             as_Register($src$$reg),
12895             r, s);
12896   %}
12897 
12898   ins_pipe(ialu_reg_shift);
12899 %}
12900 
12901 // This pattern is automatically generated from aarch64_ad.m4
12902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12903 
12904 // Shift Left followed by Shift Right.
12905 // This idiom is used by the compiler for the i2b bytecode etc.
12906 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12907 %{
12908   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12909   ins_cost(INSN_COST * 2);
12910   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12911   ins_encode %{
12912     int lshift = $lshift_count$$constant & 31;
12913     int rshift = $rshift_count$$constant & 31;
12914     int s = 31 - lshift;
12915     int r = (rshift - lshift) & 31;
12916     __ sbfmw(as_Register($dst$$reg),
12917             as_Register($src$$reg),
12918             r, s);
12919   %}
12920 
12921   ins_pipe(ialu_reg_shift);
12922 %}
12923 
12924 // This pattern is automatically generated from aarch64_ad.m4
12925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12926 
12927 // Shift Left followed by Shift Right.
12928 // This idiom is used by the compiler for the i2b bytecode etc.
12929 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12930 %{
12931   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12932   ins_cost(INSN_COST * 2);
12933   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12934   ins_encode %{
12935     int lshift = $lshift_count$$constant & 63;
12936     int rshift = $rshift_count$$constant & 63;
12937     int s = 63 - lshift;
12938     int r = (rshift - lshift) & 63;
12939     __ ubfm(as_Register($dst$$reg),
12940             as_Register($src$$reg),
12941             r, s);
12942   %}
12943 
12944   ins_pipe(ialu_reg_shift);
12945 %}
12946 
12947 // This pattern is automatically generated from aarch64_ad.m4
12948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12949 
12950 // Shift Left followed by Shift Right.
12951 // This idiom is used by the compiler for the i2b bytecode etc.
12952 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12953 %{
12954   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12955   ins_cost(INSN_COST * 2);
12956   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12957   ins_encode %{
12958     int lshift = $lshift_count$$constant & 31;
12959     int rshift = $rshift_count$$constant & 31;
12960     int s = 31 - lshift;
12961     int r = (rshift - lshift) & 31;
12962     __ ubfmw(as_Register($dst$$reg),
12963             as_Register($src$$reg),
12964             r, s);
12965   %}
12966 
12967   ins_pipe(ialu_reg_shift);
12968 %}
12969 
12970 // Bitfield extract with shift & mask
12971 
12972 // This pattern is automatically generated from aarch64_ad.m4
12973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12974 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12975 %{
12976   match(Set dst (AndI (URShiftI src rshift) mask));
12977   // Make sure we are not going to exceed what ubfxw can do.
12978   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12979 
12980   ins_cost(INSN_COST);
12981   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12982   ins_encode %{
12983     int rshift = $rshift$$constant & 31;
12984     intptr_t mask = $mask$$constant;
12985     int width = exact_log2(mask+1);
12986     __ ubfxw(as_Register($dst$$reg),
12987             as_Register($src$$reg), rshift, width);
12988   %}
12989   ins_pipe(ialu_reg_shift);
12990 %}
12991 
12992 // This pattern is automatically generated from aarch64_ad.m4
12993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12994 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12995 %{
12996   match(Set dst (AndL (URShiftL src rshift) mask));
12997   // Make sure we are not going to exceed what ubfx can do.
12998   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12999 
13000   ins_cost(INSN_COST);
13001   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13002   ins_encode %{
13003     int rshift = $rshift$$constant & 63;
13004     intptr_t mask = $mask$$constant;
13005     int width = exact_log2_long(mask+1);
13006     __ ubfx(as_Register($dst$$reg),
13007             as_Register($src$$reg), rshift, width);
13008   %}
13009   ins_pipe(ialu_reg_shift);
13010 %}
13011 
13012 
13013 // This pattern is automatically generated from aarch64_ad.m4
13014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13015 
13016 // We can use ubfx when extending an And with a mask when we know mask
13017 // is positive.  We know that because immI_bitmask guarantees it.
13018 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13019 %{
13020   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13021   // Make sure we are not going to exceed what ubfxw can do.
13022   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13023 
13024   ins_cost(INSN_COST * 2);
13025   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13026   ins_encode %{
13027     int rshift = $rshift$$constant & 31;
13028     intptr_t mask = $mask$$constant;
13029     int width = exact_log2(mask+1);
13030     __ ubfx(as_Register($dst$$reg),
13031             as_Register($src$$reg), rshift, width);
13032   %}
13033   ins_pipe(ialu_reg_shift);
13034 %}
13035 
13036 
13037 // This pattern is automatically generated from aarch64_ad.m4
13038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13039 
13040 // We can use ubfiz when masking by a positive number and then left shifting the result.
13041 // We know that the mask is positive because immI_bitmask guarantees it.
13042 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13043 %{
13044   match(Set dst (LShiftI (AndI src mask) lshift));
13045   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13046 
13047   ins_cost(INSN_COST);
13048   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13049   ins_encode %{
13050     int lshift = $lshift$$constant & 31;
13051     intptr_t mask = $mask$$constant;
13052     int width = exact_log2(mask+1);
13053     __ ubfizw(as_Register($dst$$reg),
13054           as_Register($src$$reg), lshift, width);
13055   %}
13056   ins_pipe(ialu_reg_shift);
13057 %}
13058 
13059 // This pattern is automatically generated from aarch64_ad.m4
13060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13061 
13062 // We can use ubfiz when masking by a positive number and then left shifting the result.
13063 // We know that the mask is positive because immL_bitmask guarantees it.
13064 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13065 %{
13066   match(Set dst (LShiftL (AndL src mask) lshift));
13067   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13068 
13069   ins_cost(INSN_COST);
13070   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13071   ins_encode %{
13072     int lshift = $lshift$$constant & 63;
13073     intptr_t mask = $mask$$constant;
13074     int width = exact_log2_long(mask+1);
13075     __ ubfiz(as_Register($dst$$reg),
13076           as_Register($src$$reg), lshift, width);
13077   %}
13078   ins_pipe(ialu_reg_shift);
13079 %}
13080 
13081 // This pattern is automatically generated from aarch64_ad.m4
13082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13083 
13084 // We can use ubfiz when masking by a positive number and then left shifting the result.
13085 // We know that the mask is positive because immI_bitmask guarantees it.
13086 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13087 %{
13088   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13089   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13090 
13091   ins_cost(INSN_COST);
13092   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13093   ins_encode %{
13094     int lshift = $lshift$$constant & 31;
13095     intptr_t mask = $mask$$constant;
13096     int width = exact_log2(mask+1);
13097     __ ubfizw(as_Register($dst$$reg),
13098           as_Register($src$$reg), lshift, width);
13099   %}
13100   ins_pipe(ialu_reg_shift);
13101 %}
13102 
13103 // This pattern is automatically generated from aarch64_ad.m4
13104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13105 
13106 // We can use ubfiz when masking by a positive number and then left shifting the result.
13107 // We know that the mask is positive because immL_bitmask guarantees it.
13108 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13109 %{
13110   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13111   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13112 
13113   ins_cost(INSN_COST);
13114   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13115   ins_encode %{
13116     int lshift = $lshift$$constant & 63;
13117     intptr_t mask = $mask$$constant;
13118     int width = exact_log2_long(mask+1);
13119     __ ubfiz(as_Register($dst$$reg),
13120           as_Register($src$$reg), lshift, width);
13121   %}
13122   ins_pipe(ialu_reg_shift);
13123 %}
13124 
13125 
13126 // This pattern is automatically generated from aarch64_ad.m4
13127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13128 
13129 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13130 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13131 %{
13132   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13133   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13134 
13135   ins_cost(INSN_COST);
13136   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13137   ins_encode %{
13138     int lshift = $lshift$$constant & 63;
13139     intptr_t mask = $mask$$constant;
13140     int width = exact_log2(mask+1);
13141     __ ubfiz(as_Register($dst$$reg),
13142              as_Register($src$$reg), lshift, width);
13143   %}
13144   ins_pipe(ialu_reg_shift);
13145 %}
13146 
13147 // This pattern is automatically generated from aarch64_ad.m4
13148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13149 
13150 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13151 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13152 %{
13153   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13154   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13155 
13156   ins_cost(INSN_COST);
13157   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13158   ins_encode %{
13159     int lshift = $lshift$$constant & 31;
13160     intptr_t mask = $mask$$constant;
13161     int width = exact_log2(mask+1);
13162     __ ubfiz(as_Register($dst$$reg),
13163              as_Register($src$$reg), lshift, width);
13164   %}
13165   ins_pipe(ialu_reg_shift);
13166 %}
13167 
13168 // This pattern is automatically generated from aarch64_ad.m4
13169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13170 
13171 // Can skip int2long conversions after AND with small bitmask
13172 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13173 %{
13174   match(Set dst (ConvI2L (AndI src msk)));
13175   ins_cost(INSN_COST);
13176   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13177   ins_encode %{
13178     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13179   %}
13180   ins_pipe(ialu_reg_shift);
13181 %}
13182 
13183 
13184 // Rotations
13185 // This pattern is automatically generated from aarch64_ad.m4
13186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13187 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13188 %{
13189   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13190   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13191 
13192   ins_cost(INSN_COST);
13193   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13194 
13195   ins_encode %{
13196     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13197             $rshift$$constant & 63);
13198   %}
13199   ins_pipe(ialu_reg_reg_extr);
13200 %}
13201 
13202 
13203 // This pattern is automatically generated from aarch64_ad.m4
13204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13205 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13206 %{
13207   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13208   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13209 
13210   ins_cost(INSN_COST);
13211   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13212 
13213   ins_encode %{
13214     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13215             $rshift$$constant & 31);
13216   %}
13217   ins_pipe(ialu_reg_reg_extr);
13218 %}
13219 
13220 
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13224 %{
13225   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13226   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13227 
13228   ins_cost(INSN_COST);
13229   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13230 
13231   ins_encode %{
13232     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13233             $rshift$$constant & 63);
13234   %}
13235   ins_pipe(ialu_reg_reg_extr);
13236 %}
13237 
13238 
13239 // This pattern is automatically generated from aarch64_ad.m4
13240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13241 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13242 %{
13243   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13244   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13245 
13246   ins_cost(INSN_COST);
13247   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13248 
13249   ins_encode %{
13250     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13251             $rshift$$constant & 31);
13252   %}
13253   ins_pipe(ialu_reg_reg_extr);
13254 %}
13255 
13256 
13257 // This pattern is automatically generated from aarch64_ad.m4
13258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13259 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13260 %{
13261   match(Set dst (RotateRight src shift));
13262 
13263   ins_cost(INSN_COST);
13264   format %{ "ror    $dst, $src, $shift" %}
13265 
13266   ins_encode %{
13267      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13268                $shift$$constant & 0x1f);
13269   %}
13270   ins_pipe(ialu_reg_reg_vshift);
13271 %}
13272 
13273 // This pattern is automatically generated from aarch64_ad.m4
13274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13275 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13276 %{
13277   match(Set dst (RotateRight src shift));
13278 
13279   ins_cost(INSN_COST);
13280   format %{ "ror    $dst, $src, $shift" %}
13281 
13282   ins_encode %{
13283      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13284                $shift$$constant & 0x3f);
13285   %}
13286   ins_pipe(ialu_reg_reg_vshift);
13287 %}
13288 
13289 // This pattern is automatically generated from aarch64_ad.m4
13290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13291 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13292 %{
13293   match(Set dst (RotateRight src shift));
13294 
13295   ins_cost(INSN_COST);
13296   format %{ "ror    $dst, $src, $shift" %}
13297 
13298   ins_encode %{
13299      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13300   %}
13301   ins_pipe(ialu_reg_reg_vshift);
13302 %}
13303 
13304 // This pattern is automatically generated from aarch64_ad.m4
13305 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13306 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13307 %{
13308   match(Set dst (RotateRight src shift));
13309 
13310   ins_cost(INSN_COST);
13311   format %{ "ror    $dst, $src, $shift" %}
13312 
13313   ins_encode %{
13314      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13315   %}
13316   ins_pipe(ialu_reg_reg_vshift);
13317 %}
13318 
13319 // This pattern is automatically generated from aarch64_ad.m4
13320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13321 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13322 %{
13323   match(Set dst (RotateLeft src shift));
13324 
13325   ins_cost(INSN_COST);
13326   format %{ "rol    $dst, $src, $shift" %}
13327 
13328   ins_encode %{
13329      __ subw(rscratch1, zr, as_Register($shift$$reg));
13330      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13331   %}
13332   ins_pipe(ialu_reg_reg_vshift);
13333 %}
13334 
13335 // This pattern is automatically generated from aarch64_ad.m4
13336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13337 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13338 %{
13339   match(Set dst (RotateLeft src shift));
13340 
13341   ins_cost(INSN_COST);
13342   format %{ "rol    $dst, $src, $shift" %}
13343 
13344   ins_encode %{
13345      __ subw(rscratch1, zr, as_Register($shift$$reg));
13346      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13347   %}
13348   ins_pipe(ialu_reg_reg_vshift);
13349 %}
13350 
13351 
13352 // Add/subtract (extended)
13353 
13354 // This pattern is automatically generated from aarch64_ad.m4
13355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13356 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13357 %{
13358   match(Set dst (AddL src1 (ConvI2L src2)));
13359   ins_cost(INSN_COST);
13360   format %{ "add  $dst, $src1, $src2, sxtw" %}
13361 
13362    ins_encode %{
13363      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13364             as_Register($src2$$reg), ext::sxtw);
13365    %}
13366   ins_pipe(ialu_reg_reg);
13367 %}
13368 
13369 // This pattern is automatically generated from aarch64_ad.m4
13370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13371 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13372 %{
13373   match(Set dst (SubL src1 (ConvI2L src2)));
13374   ins_cost(INSN_COST);
13375   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13376 
13377    ins_encode %{
13378      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13379             as_Register($src2$$reg), ext::sxtw);
13380    %}
13381   ins_pipe(ialu_reg_reg);
13382 %}
13383 
13384 // This pattern is automatically generated from aarch64_ad.m4
13385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13386 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13387 %{
13388   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13389   ins_cost(INSN_COST);
13390   format %{ "add  $dst, $src1, $src2, sxth" %}
13391 
13392    ins_encode %{
13393      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13394             as_Register($src2$$reg), ext::sxth);
13395    %}
13396   ins_pipe(ialu_reg_reg);
13397 %}
13398 
13399 // This pattern is automatically generated from aarch64_ad.m4
13400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13401 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13402 %{
13403   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13404   ins_cost(INSN_COST);
13405   format %{ "add  $dst, $src1, $src2, sxtb" %}
13406 
13407    ins_encode %{
13408      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13409             as_Register($src2$$reg), ext::sxtb);
13410    %}
13411   ins_pipe(ialu_reg_reg);
13412 %}
13413 
13414 // This pattern is automatically generated from aarch64_ad.m4
13415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13416 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13417 %{
13418   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13419   ins_cost(INSN_COST);
13420   format %{ "add  $dst, $src1, $src2, uxtb" %}
13421 
13422    ins_encode %{
13423      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13424             as_Register($src2$$reg), ext::uxtb);
13425    %}
13426   ins_pipe(ialu_reg_reg);
13427 %}
13428 
13429 // This pattern is automatically generated from aarch64_ad.m4
13430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13431 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13432 %{
13433   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13434   ins_cost(INSN_COST);
13435   format %{ "add  $dst, $src1, $src2, sxth" %}
13436 
13437    ins_encode %{
13438      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13439             as_Register($src2$$reg), ext::sxth);
13440    %}
13441   ins_pipe(ialu_reg_reg);
13442 %}
13443 
13444 // This pattern is automatically generated from aarch64_ad.m4
13445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13446 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13447 %{
13448   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13449   ins_cost(INSN_COST);
13450   format %{ "add  $dst, $src1, $src2, sxtw" %}
13451 
13452    ins_encode %{
13453      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13454             as_Register($src2$$reg), ext::sxtw);
13455    %}
13456   ins_pipe(ialu_reg_reg);
13457 %}
13458 
13459 // This pattern is automatically generated from aarch64_ad.m4
13460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13461 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13462 %{
13463   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13464   ins_cost(INSN_COST);
13465   format %{ "add  $dst, $src1, $src2, sxtb" %}
13466 
13467    ins_encode %{
13468      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13469             as_Register($src2$$reg), ext::sxtb);
13470    %}
13471   ins_pipe(ialu_reg_reg);
13472 %}
13473 
13474 // This pattern is automatically generated from aarch64_ad.m4
13475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13476 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13477 %{
13478   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13479   ins_cost(INSN_COST);
13480   format %{ "add  $dst, $src1, $src2, uxtb" %}
13481 
13482    ins_encode %{
13483      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13484             as_Register($src2$$reg), ext::uxtb);
13485    %}
13486   ins_pipe(ialu_reg_reg);
13487 %}
13488 
13489 // This pattern is automatically generated from aarch64_ad.m4
13490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13491 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13492 %{
13493   match(Set dst (AddI src1 (AndI src2 mask)));
13494   ins_cost(INSN_COST);
13495   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13496 
13497    ins_encode %{
13498      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13499             as_Register($src2$$reg), ext::uxtb);
13500    %}
13501   ins_pipe(ialu_reg_reg);
13502 %}
13503 
13504 // This pattern is automatically generated from aarch64_ad.m4
13505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13506 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13507 %{
13508   match(Set dst (AddI src1 (AndI src2 mask)));
13509   ins_cost(INSN_COST);
13510   format %{ "addw  $dst, $src1, $src2, uxth" %}
13511 
13512    ins_encode %{
13513      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13514             as_Register($src2$$reg), ext::uxth);
13515    %}
13516   ins_pipe(ialu_reg_reg);
13517 %}
13518 
13519 // This pattern is automatically generated from aarch64_ad.m4
13520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13521 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13522 %{
13523   match(Set dst (AddL src1 (AndL src2 mask)));
13524   ins_cost(INSN_COST);
13525   format %{ "add  $dst, $src1, $src2, uxtb" %}
13526 
13527    ins_encode %{
13528      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13529             as_Register($src2$$reg), ext::uxtb);
13530    %}
13531   ins_pipe(ialu_reg_reg);
13532 %}
13533 
13534 // This pattern is automatically generated from aarch64_ad.m4
13535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13536 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13537 %{
13538   match(Set dst (AddL src1 (AndL src2 mask)));
13539   ins_cost(INSN_COST);
13540   format %{ "add  $dst, $src1, $src2, uxth" %}
13541 
13542    ins_encode %{
13543      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13544             as_Register($src2$$reg), ext::uxth);
13545    %}
13546   ins_pipe(ialu_reg_reg);
13547 %}
13548 
13549 // This pattern is automatically generated from aarch64_ad.m4
13550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13551 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13552 %{
13553   match(Set dst (AddL src1 (AndL src2 mask)));
13554   ins_cost(INSN_COST);
13555   format %{ "add  $dst, $src1, $src2, uxtw" %}
13556 
13557    ins_encode %{
13558      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13559             as_Register($src2$$reg), ext::uxtw);
13560    %}
13561   ins_pipe(ialu_reg_reg);
13562 %}
13563 
13564 // This pattern is automatically generated from aarch64_ad.m4
13565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13566 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13567 %{
13568   match(Set dst (SubI src1 (AndI src2 mask)));
13569   ins_cost(INSN_COST);
13570   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13571 
13572    ins_encode %{
13573      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13574             as_Register($src2$$reg), ext::uxtb);
13575    %}
13576   ins_pipe(ialu_reg_reg);
13577 %}
13578 
13579 // This pattern is automatically generated from aarch64_ad.m4
13580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13581 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13582 %{
13583   match(Set dst (SubI src1 (AndI src2 mask)));
13584   ins_cost(INSN_COST);
13585   format %{ "subw  $dst, $src1, $src2, uxth" %}
13586 
13587    ins_encode %{
13588      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13589             as_Register($src2$$reg), ext::uxth);
13590    %}
13591   ins_pipe(ialu_reg_reg);
13592 %}
13593 
13594 // This pattern is automatically generated from aarch64_ad.m4
13595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13596 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13597 %{
13598   match(Set dst (SubL src1 (AndL src2 mask)));
13599   ins_cost(INSN_COST);
13600   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13601 
13602    ins_encode %{
13603      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13604             as_Register($src2$$reg), ext::uxtb);
13605    %}
13606   ins_pipe(ialu_reg_reg);
13607 %}
13608 
13609 // This pattern is automatically generated from aarch64_ad.m4
13610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13611 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13612 %{
13613   match(Set dst (SubL src1 (AndL src2 mask)));
13614   ins_cost(INSN_COST);
13615   format %{ "sub  $dst, $src1, $src2, uxth" %}
13616 
13617    ins_encode %{
13618      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13619             as_Register($src2$$reg), ext::uxth);
13620    %}
13621   ins_pipe(ialu_reg_reg);
13622 %}
13623 
13624 // This pattern is automatically generated from aarch64_ad.m4
13625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13626 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13627 %{
13628   match(Set dst (SubL src1 (AndL src2 mask)));
13629   ins_cost(INSN_COST);
13630   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13631 
13632    ins_encode %{
13633      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13634             as_Register($src2$$reg), ext::uxtw);
13635    %}
13636   ins_pipe(ialu_reg_reg);
13637 %}
13638 
13639 
13640 // This pattern is automatically generated from aarch64_ad.m4
13641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13642 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13643 %{
13644   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13645   ins_cost(1.9 * INSN_COST);
13646   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13647 
13648    ins_encode %{
13649      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13650             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13651    %}
13652   ins_pipe(ialu_reg_reg_shift);
13653 %}
13654 
13655 // This pattern is automatically generated from aarch64_ad.m4
13656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13657 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13658 %{
13659   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13660   ins_cost(1.9 * INSN_COST);
13661   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13662 
13663    ins_encode %{
13664      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13665             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13666    %}
13667   ins_pipe(ialu_reg_reg_shift);
13668 %}
13669 
13670 // This pattern is automatically generated from aarch64_ad.m4
13671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13672 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13673 %{
13674   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13675   ins_cost(1.9 * INSN_COST);
13676   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13677 
13678    ins_encode %{
13679      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13680             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13681    %}
13682   ins_pipe(ialu_reg_reg_shift);
13683 %}
13684 
13685 // This pattern is automatically generated from aarch64_ad.m4
13686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13687 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13688 %{
13689   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13690   ins_cost(1.9 * INSN_COST);
13691   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13692 
13693    ins_encode %{
13694      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13695             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13696    %}
13697   ins_pipe(ialu_reg_reg_shift);
13698 %}
13699 
13700 // This pattern is automatically generated from aarch64_ad.m4
13701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13702 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13703 %{
13704   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13705   ins_cost(1.9 * INSN_COST);
13706   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13707 
13708    ins_encode %{
13709      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13710             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13711    %}
13712   ins_pipe(ialu_reg_reg_shift);
13713 %}
13714 
13715 // This pattern is automatically generated from aarch64_ad.m4
13716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13717 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13718 %{
13719   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13720   ins_cost(1.9 * INSN_COST);
13721   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13722 
13723    ins_encode %{
13724      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13725             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13726    %}
13727   ins_pipe(ialu_reg_reg_shift);
13728 %}
13729 
13730 // This pattern is automatically generated from aarch64_ad.m4
13731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13732 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13733 %{
13734   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13735   ins_cost(1.9 * INSN_COST);
13736   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13737 
13738    ins_encode %{
13739      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13740             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13741    %}
13742   ins_pipe(ialu_reg_reg_shift);
13743 %}
13744 
13745 // This pattern is automatically generated from aarch64_ad.m4
13746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13747 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13748 %{
13749   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13750   ins_cost(1.9 * INSN_COST);
13751   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13752 
13753    ins_encode %{
13754      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13755             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13756    %}
13757   ins_pipe(ialu_reg_reg_shift);
13758 %}
13759 
13760 // This pattern is automatically generated from aarch64_ad.m4
13761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13762 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13763 %{
13764   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13765   ins_cost(1.9 * INSN_COST);
13766   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13767 
13768    ins_encode %{
13769      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13770             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13771    %}
13772   ins_pipe(ialu_reg_reg_shift);
13773 %}
13774 
13775 // This pattern is automatically generated from aarch64_ad.m4
13776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13777 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13778 %{
13779   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13780   ins_cost(1.9 * INSN_COST);
13781   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13782 
13783    ins_encode %{
13784      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13785             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13786    %}
13787   ins_pipe(ialu_reg_reg_shift);
13788 %}
13789 
13790 // This pattern is automatically generated from aarch64_ad.m4
13791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13792 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13793 %{
13794   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13795   ins_cost(1.9 * INSN_COST);
13796   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13797 
13798    ins_encode %{
13799      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13800             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13801    %}
13802   ins_pipe(ialu_reg_reg_shift);
13803 %}
13804 
13805 // This pattern is automatically generated from aarch64_ad.m4
13806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13807 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13808 %{
13809   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13810   ins_cost(1.9 * INSN_COST);
13811   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13812 
13813    ins_encode %{
13814      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13815             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13816    %}
13817   ins_pipe(ialu_reg_reg_shift);
13818 %}
13819 
13820 // This pattern is automatically generated from aarch64_ad.m4
13821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13822 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13823 %{
13824   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13825   ins_cost(1.9 * INSN_COST);
13826   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13827 
13828    ins_encode %{
13829      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13830             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13831    %}
13832   ins_pipe(ialu_reg_reg_shift);
13833 %}
13834 
13835 // This pattern is automatically generated from aarch64_ad.m4
13836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13837 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13838 %{
13839   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13840   ins_cost(1.9 * INSN_COST);
13841   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13842 
13843    ins_encode %{
13844      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13845             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13846    %}
13847   ins_pipe(ialu_reg_reg_shift);
13848 %}
13849 
13850 // This pattern is automatically generated from aarch64_ad.m4
13851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13852 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13853 %{
13854   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13855   ins_cost(1.9 * INSN_COST);
13856   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13857 
13858    ins_encode %{
13859      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13860             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13861    %}
13862   ins_pipe(ialu_reg_reg_shift);
13863 %}
13864 
13865 // This pattern is automatically generated from aarch64_ad.m4
13866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13867 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13868 %{
13869   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13870   ins_cost(1.9 * INSN_COST);
13871   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13872 
13873    ins_encode %{
13874      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13875             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13876    %}
13877   ins_pipe(ialu_reg_reg_shift);
13878 %}
13879 
13880 // This pattern is automatically generated from aarch64_ad.m4
13881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13882 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13883 %{
13884   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13885   ins_cost(1.9 * INSN_COST);
13886   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13887 
13888    ins_encode %{
13889      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13890             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13891    %}
13892   ins_pipe(ialu_reg_reg_shift);
13893 %}
13894 
13895 // This pattern is automatically generated from aarch64_ad.m4
13896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13897 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13898 %{
13899   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13900   ins_cost(1.9 * INSN_COST);
13901   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13902 
13903    ins_encode %{
13904      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13905             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13906    %}
13907   ins_pipe(ialu_reg_reg_shift);
13908 %}
13909 
13910 // This pattern is automatically generated from aarch64_ad.m4
13911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13912 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13913 %{
13914   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13915   ins_cost(1.9 * INSN_COST);
13916   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13917 
13918    ins_encode %{
13919      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13920             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13921    %}
13922   ins_pipe(ialu_reg_reg_shift);
13923 %}
13924 
13925 // This pattern is automatically generated from aarch64_ad.m4
13926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13927 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13928 %{
13929   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13930   ins_cost(1.9 * INSN_COST);
13931   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13932 
13933    ins_encode %{
13934      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13935             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13936    %}
13937   ins_pipe(ialu_reg_reg_shift);
13938 %}
13939 
13940 // This pattern is automatically generated from aarch64_ad.m4
13941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13942 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13943 %{
13944   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13945   ins_cost(1.9 * INSN_COST);
13946   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13947 
13948    ins_encode %{
13949      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13950             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13951    %}
13952   ins_pipe(ialu_reg_reg_shift);
13953 %}
13954 
13955 // This pattern is automatically generated from aarch64_ad.m4
13956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13957 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13958 %{
13959   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13960   ins_cost(1.9 * INSN_COST);
13961   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13962 
13963    ins_encode %{
13964      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13965             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13966    %}
13967   ins_pipe(ialu_reg_reg_shift);
13968 %}
13969 
13970 
13971 
13972 // END This section of the file is automatically generated. Do not edit --------------
13973 
13974 
13975 // ============================================================================
13976 // Floating Point Arithmetic Instructions
13977 
13978 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13979   match(Set dst (AddF src1 src2));
13980 
13981   ins_cost(INSN_COST * 5);
13982   format %{ "fadds   $dst, $src1, $src2" %}
13983 
13984   ins_encode %{
13985     __ fadds(as_FloatRegister($dst$$reg),
13986              as_FloatRegister($src1$$reg),
13987              as_FloatRegister($src2$$reg));
13988   %}
13989 
13990   ins_pipe(fp_dop_reg_reg_s);
13991 %}
13992 
13993 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13994   match(Set dst (AddD src1 src2));
13995 
13996   ins_cost(INSN_COST * 5);
13997   format %{ "faddd   $dst, $src1, $src2" %}
13998 
13999   ins_encode %{
14000     __ faddd(as_FloatRegister($dst$$reg),
14001              as_FloatRegister($src1$$reg),
14002              as_FloatRegister($src2$$reg));
14003   %}
14004 
14005   ins_pipe(fp_dop_reg_reg_d);
14006 %}
14007 
14008 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14009   match(Set dst (SubF src1 src2));
14010 
14011   ins_cost(INSN_COST * 5);
14012   format %{ "fsubs   $dst, $src1, $src2" %}
14013 
14014   ins_encode %{
14015     __ fsubs(as_FloatRegister($dst$$reg),
14016              as_FloatRegister($src1$$reg),
14017              as_FloatRegister($src2$$reg));
14018   %}
14019 
14020   ins_pipe(fp_dop_reg_reg_s);
14021 %}
14022 
14023 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14024   match(Set dst (SubD src1 src2));
14025 
14026   ins_cost(INSN_COST * 5);
14027   format %{ "fsubd   $dst, $src1, $src2" %}
14028 
14029   ins_encode %{
14030     __ fsubd(as_FloatRegister($dst$$reg),
14031              as_FloatRegister($src1$$reg),
14032              as_FloatRegister($src2$$reg));
14033   %}
14034 
14035   ins_pipe(fp_dop_reg_reg_d);
14036 %}
14037 
14038 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14039   match(Set dst (MulF src1 src2));
14040 
14041   ins_cost(INSN_COST * 6);
14042   format %{ "fmuls   $dst, $src1, $src2" %}
14043 
14044   ins_encode %{
14045     __ fmuls(as_FloatRegister($dst$$reg),
14046              as_FloatRegister($src1$$reg),
14047              as_FloatRegister($src2$$reg));
14048   %}
14049 
14050   ins_pipe(fp_dop_reg_reg_s);
14051 %}
14052 
14053 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14054   match(Set dst (MulD src1 src2));
14055 
14056   ins_cost(INSN_COST * 6);
14057   format %{ "fmuld   $dst, $src1, $src2" %}
14058 
14059   ins_encode %{
14060     __ fmuld(as_FloatRegister($dst$$reg),
14061              as_FloatRegister($src1$$reg),
14062              as_FloatRegister($src2$$reg));
14063   %}
14064 
14065   ins_pipe(fp_dop_reg_reg_d);
14066 %}
14067 
14068 // src1 * src2 + src3
14069 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14070   predicate(UseFMA);
14071   match(Set dst (FmaF src3 (Binary src1 src2)));
14072 
14073   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14074 
14075   ins_encode %{
14076     __ fmadds(as_FloatRegister($dst$$reg),
14077              as_FloatRegister($src1$$reg),
14078              as_FloatRegister($src2$$reg),
14079              as_FloatRegister($src3$$reg));
14080   %}
14081 
14082   ins_pipe(pipe_class_default);
14083 %}
14084 
14085 // src1 * src2 + src3
14086 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14087   predicate(UseFMA);
14088   match(Set dst (FmaD src3 (Binary src1 src2)));
14089 
14090   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14091 
14092   ins_encode %{
14093     __ fmaddd(as_FloatRegister($dst$$reg),
14094              as_FloatRegister($src1$$reg),
14095              as_FloatRegister($src2$$reg),
14096              as_FloatRegister($src3$$reg));
14097   %}
14098 
14099   ins_pipe(pipe_class_default);
14100 %}
14101 
14102 // -src1 * src2 + src3
14103 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14104   predicate(UseFMA);
14105   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14106   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14107 
14108   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14109 
14110   ins_encode %{
14111     __ fmsubs(as_FloatRegister($dst$$reg),
14112               as_FloatRegister($src1$$reg),
14113               as_FloatRegister($src2$$reg),
14114               as_FloatRegister($src3$$reg));
14115   %}
14116 
14117   ins_pipe(pipe_class_default);
14118 %}
14119 
14120 // -src1 * src2 + src3
14121 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14122   predicate(UseFMA);
14123   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14124   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14125 
14126   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14127 
14128   ins_encode %{
14129     __ fmsubd(as_FloatRegister($dst$$reg),
14130               as_FloatRegister($src1$$reg),
14131               as_FloatRegister($src2$$reg),
14132               as_FloatRegister($src3$$reg));
14133   %}
14134 
14135   ins_pipe(pipe_class_default);
14136 %}
14137 
14138 // -src1 * src2 - src3
14139 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14140   predicate(UseFMA);
14141   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14142   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14143 
14144   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14145 
14146   ins_encode %{
14147     __ fnmadds(as_FloatRegister($dst$$reg),
14148                as_FloatRegister($src1$$reg),
14149                as_FloatRegister($src2$$reg),
14150                as_FloatRegister($src3$$reg));
14151   %}
14152 
14153   ins_pipe(pipe_class_default);
14154 %}
14155 
14156 // -src1 * src2 - src3
14157 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14158   predicate(UseFMA);
14159   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14160   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14161 
14162   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14163 
14164   ins_encode %{
14165     __ fnmaddd(as_FloatRegister($dst$$reg),
14166                as_FloatRegister($src1$$reg),
14167                as_FloatRegister($src2$$reg),
14168                as_FloatRegister($src3$$reg));
14169   %}
14170 
14171   ins_pipe(pipe_class_default);
14172 %}
14173 
14174 // src1 * src2 - src3
14175 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14176   predicate(UseFMA);
14177   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14178 
14179   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14180 
14181   ins_encode %{
14182     __ fnmsubs(as_FloatRegister($dst$$reg),
14183                as_FloatRegister($src1$$reg),
14184                as_FloatRegister($src2$$reg),
14185                as_FloatRegister($src3$$reg));
14186   %}
14187 
14188   ins_pipe(pipe_class_default);
14189 %}
14190 
14191 // src1 * src2 - src3
14192 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14193   predicate(UseFMA);
14194   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14195 
14196   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14197 
14198   ins_encode %{
14199   // n.b. insn name should be fnmsubd
14200     __ fnmsub(as_FloatRegister($dst$$reg),
14201               as_FloatRegister($src1$$reg),
14202               as_FloatRegister($src2$$reg),
14203               as_FloatRegister($src3$$reg));
14204   %}
14205 
14206   ins_pipe(pipe_class_default);
14207 %}
14208 
14209 
14210 // Math.max(FF)F
14211 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14212   match(Set dst (MaxF src1 src2));
14213 
14214   format %{ "fmaxs   $dst, $src1, $src2" %}
14215   ins_encode %{
14216     __ fmaxs(as_FloatRegister($dst$$reg),
14217              as_FloatRegister($src1$$reg),
14218              as_FloatRegister($src2$$reg));
14219   %}
14220 
14221   ins_pipe(fp_dop_reg_reg_s);
14222 %}
14223 
14224 // Math.min(FF)F
14225 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14226   match(Set dst (MinF src1 src2));
14227 
14228   format %{ "fmins   $dst, $src1, $src2" %}
14229   ins_encode %{
14230     __ fmins(as_FloatRegister($dst$$reg),
14231              as_FloatRegister($src1$$reg),
14232              as_FloatRegister($src2$$reg));
14233   %}
14234 
14235   ins_pipe(fp_dop_reg_reg_s);
14236 %}
14237 
14238 // Math.max(DD)D
14239 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14240   match(Set dst (MaxD src1 src2));
14241 
14242   format %{ "fmaxd   $dst, $src1, $src2" %}
14243   ins_encode %{
14244     __ fmaxd(as_FloatRegister($dst$$reg),
14245              as_FloatRegister($src1$$reg),
14246              as_FloatRegister($src2$$reg));
14247   %}
14248 
14249   ins_pipe(fp_dop_reg_reg_d);
14250 %}
14251 
14252 // Math.min(DD)D
14253 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14254   match(Set dst (MinD src1 src2));
14255 
14256   format %{ "fmind   $dst, $src1, $src2" %}
14257   ins_encode %{
14258     __ fmind(as_FloatRegister($dst$$reg),
14259              as_FloatRegister($src1$$reg),
14260              as_FloatRegister($src2$$reg));
14261   %}
14262 
14263   ins_pipe(fp_dop_reg_reg_d);
14264 %}
14265 
14266 
14267 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14268   match(Set dst (DivF src1  src2));
14269 
14270   ins_cost(INSN_COST * 18);
14271   format %{ "fdivs   $dst, $src1, $src2" %}
14272 
14273   ins_encode %{
14274     __ fdivs(as_FloatRegister($dst$$reg),
14275              as_FloatRegister($src1$$reg),
14276              as_FloatRegister($src2$$reg));
14277   %}
14278 
14279   ins_pipe(fp_div_s);
14280 %}
14281 
14282 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14283   match(Set dst (DivD src1  src2));
14284 
14285   ins_cost(INSN_COST * 32);
14286   format %{ "fdivd   $dst, $src1, $src2" %}
14287 
14288   ins_encode %{
14289     __ fdivd(as_FloatRegister($dst$$reg),
14290              as_FloatRegister($src1$$reg),
14291              as_FloatRegister($src2$$reg));
14292   %}
14293 
14294   ins_pipe(fp_div_d);
14295 %}
14296 
14297 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14298   match(Set dst (NegF src));
14299 
14300   ins_cost(INSN_COST * 3);
14301   format %{ "fneg   $dst, $src" %}
14302 
14303   ins_encode %{
14304     __ fnegs(as_FloatRegister($dst$$reg),
14305              as_FloatRegister($src$$reg));
14306   %}
14307 
14308   ins_pipe(fp_uop_s);
14309 %}
14310 
14311 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14312   match(Set dst (NegD src));
14313 
14314   ins_cost(INSN_COST * 3);
14315   format %{ "fnegd   $dst, $src" %}
14316 
14317   ins_encode %{
14318     __ fnegd(as_FloatRegister($dst$$reg),
14319              as_FloatRegister($src$$reg));
14320   %}
14321 
14322   ins_pipe(fp_uop_d);
14323 %}
14324 
14325 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14326 %{
14327   match(Set dst (AbsI src));
14328 
14329   effect(KILL cr);
14330   ins_cost(INSN_COST * 2);
14331   format %{ "cmpw  $src, zr\n\t"
14332             "cnegw $dst, $src, Assembler::LT\t# int abs"
14333   %}
14334 
14335   ins_encode %{
14336     __ cmpw(as_Register($src$$reg), zr);
14337     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14338   %}
14339   ins_pipe(pipe_class_default);
14340 %}
14341 
14342 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14343 %{
14344   match(Set dst (AbsL src));
14345 
14346   effect(KILL cr);
14347   ins_cost(INSN_COST * 2);
14348   format %{ "cmp  $src, zr\n\t"
14349             "cneg $dst, $src, Assembler::LT\t# long abs"
14350   %}
14351 
14352   ins_encode %{
14353     __ cmp(as_Register($src$$reg), zr);
14354     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14355   %}
14356   ins_pipe(pipe_class_default);
14357 %}
14358 
14359 instruct absF_reg(vRegF dst, vRegF src) %{
14360   match(Set dst (AbsF src));
14361 
14362   ins_cost(INSN_COST * 3);
14363   format %{ "fabss   $dst, $src" %}
14364   ins_encode %{
14365     __ fabss(as_FloatRegister($dst$$reg),
14366              as_FloatRegister($src$$reg));
14367   %}
14368 
14369   ins_pipe(fp_uop_s);
14370 %}
14371 
14372 instruct absD_reg(vRegD dst, vRegD src) %{
14373   match(Set dst (AbsD src));
14374 
14375   ins_cost(INSN_COST * 3);
14376   format %{ "fabsd   $dst, $src" %}
14377   ins_encode %{
14378     __ fabsd(as_FloatRegister($dst$$reg),
14379              as_FloatRegister($src$$reg));
14380   %}
14381 
14382   ins_pipe(fp_uop_d);
14383 %}
14384 
14385 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14386   match(Set dst (AbsF (SubF src1 src2)));
14387 
14388   ins_cost(INSN_COST * 3);
14389   format %{ "fabds   $dst, $src1, $src2" %}
14390   ins_encode %{
14391     __ fabds(as_FloatRegister($dst$$reg),
14392              as_FloatRegister($src1$$reg),
14393              as_FloatRegister($src2$$reg));
14394   %}
14395 
14396   ins_pipe(fp_uop_s);
14397 %}
14398 
14399 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14400   match(Set dst (AbsD (SubD src1 src2)));
14401 
14402   ins_cost(INSN_COST * 3);
14403   format %{ "fabdd   $dst, $src1, $src2" %}
14404   ins_encode %{
14405     __ fabdd(as_FloatRegister($dst$$reg),
14406              as_FloatRegister($src1$$reg),
14407              as_FloatRegister($src2$$reg));
14408   %}
14409 
14410   ins_pipe(fp_uop_d);
14411 %}
14412 
14413 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14414   match(Set dst (SqrtD src));
14415 
14416   ins_cost(INSN_COST * 50);
14417   format %{ "fsqrtd  $dst, $src" %}
14418   ins_encode %{
14419     __ fsqrtd(as_FloatRegister($dst$$reg),
14420              as_FloatRegister($src$$reg));
14421   %}
14422 
14423   ins_pipe(fp_div_s);
14424 %}
14425 
14426 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14427   match(Set dst (SqrtF src));
14428 
14429   ins_cost(INSN_COST * 50);
14430   format %{ "fsqrts  $dst, $src" %}
14431   ins_encode %{
14432     __ fsqrts(as_FloatRegister($dst$$reg),
14433              as_FloatRegister($src$$reg));
14434   %}
14435 
14436   ins_pipe(fp_div_d);
14437 %}
14438 
14439 // Math.rint, floor, ceil
14440 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14441   match(Set dst (RoundDoubleMode src rmode));
14442   format %{ "frint  $dst, $src, $rmode" %}
14443   ins_encode %{
14444     switch ($rmode$$constant) {
14445       case RoundDoubleModeNode::rmode_rint:
14446         __ frintnd(as_FloatRegister($dst$$reg),
14447                    as_FloatRegister($src$$reg));
14448         break;
14449       case RoundDoubleModeNode::rmode_floor:
14450         __ frintmd(as_FloatRegister($dst$$reg),
14451                    as_FloatRegister($src$$reg));
14452         break;
14453       case RoundDoubleModeNode::rmode_ceil:
14454         __ frintpd(as_FloatRegister($dst$$reg),
14455                    as_FloatRegister($src$$reg));
14456         break;
14457     }
14458   %}
14459   ins_pipe(fp_uop_d);
14460 %}
14461 
14462 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14463   match(Set dst (CopySignD src1 (Binary src2 zero)));
14464   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14465   format %{ "CopySignD  $dst $src1 $src2" %}
14466   ins_encode %{
14467     FloatRegister dst = as_FloatRegister($dst$$reg),
14468                   src1 = as_FloatRegister($src1$$reg),
14469                   src2 = as_FloatRegister($src2$$reg),
14470                   zero = as_FloatRegister($zero$$reg);
14471     __ fnegd(dst, zero);
14472     __ bsl(dst, __ T8B, src2, src1);
14473   %}
14474   ins_pipe(fp_uop_d);
14475 %}
14476 
14477 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14478   match(Set dst (CopySignF src1 src2));
14479   effect(TEMP_DEF dst, USE src1, USE src2);
14480   format %{ "CopySignF  $dst $src1 $src2" %}
14481   ins_encode %{
14482     FloatRegister dst = as_FloatRegister($dst$$reg),
14483                   src1 = as_FloatRegister($src1$$reg),
14484                   src2 = as_FloatRegister($src2$$reg);
14485     __ movi(dst, __ T2S, 0x80, 24);
14486     __ bsl(dst, __ T8B, src2, src1);
14487   %}
14488   ins_pipe(fp_uop_d);
14489 %}
14490 
14491 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14492   match(Set dst (SignumD src (Binary zero one)));
14493   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14494   format %{ "signumD  $dst, $src" %}
14495   ins_encode %{
14496     FloatRegister src = as_FloatRegister($src$$reg),
14497                   dst = as_FloatRegister($dst$$reg),
14498                   zero = as_FloatRegister($zero$$reg),
14499                   one = as_FloatRegister($one$$reg);
14500     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14501     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14502     // Bit selection instruction gets bit from "one" for each enabled bit in
14503     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14504     // NaN the whole "src" will be copied because "dst" is zero. For all other
14505     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14506     // from "src", and all other bits are copied from 1.0.
14507     __ bsl(dst, __ T8B, one, src);
14508   %}
14509   ins_pipe(fp_uop_d);
14510 %}
14511 
14512 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14513   match(Set dst (SignumF src (Binary zero one)));
14514   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14515   format %{ "signumF  $dst, $src" %}
14516   ins_encode %{
14517     FloatRegister src = as_FloatRegister($src$$reg),
14518                   dst = as_FloatRegister($dst$$reg),
14519                   zero = as_FloatRegister($zero$$reg),
14520                   one = as_FloatRegister($one$$reg);
14521     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14522     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14523     // Bit selection instruction gets bit from "one" for each enabled bit in
14524     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14525     // NaN the whole "src" will be copied because "dst" is zero. For all other
14526     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14527     // from "src", and all other bits are copied from 1.0.
14528     __ bsl(dst, __ T8B, one, src);
14529   %}
14530   ins_pipe(fp_uop_d);
14531 %}
14532 
14533 instruct onspinwait() %{
14534   match(OnSpinWait);
14535   ins_cost(INSN_COST);
14536 
14537   format %{ "onspinwait" %}
14538 
14539   ins_encode %{
14540     __ spin_wait();
14541   %}
14542   ins_pipe(pipe_class_empty);
14543 %}
14544 
14545 // ============================================================================
14546 // Logical Instructions
14547 
14548 // Integer Logical Instructions
14549 
14550 // And Instructions
14551 
14552 
14553 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14554   match(Set dst (AndI src1 src2));
14555 
14556   format %{ "andw  $dst, $src1, $src2\t# int" %}
14557 
14558   ins_cost(INSN_COST);
14559   ins_encode %{
14560     __ andw(as_Register($dst$$reg),
14561             as_Register($src1$$reg),
14562             as_Register($src2$$reg));
14563   %}
14564 
14565   ins_pipe(ialu_reg_reg);
14566 %}
14567 
14568 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14569   match(Set dst (AndI src1 src2));
14570 
14571   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14572 
14573   ins_cost(INSN_COST);
14574   ins_encode %{
14575     __ andw(as_Register($dst$$reg),
14576             as_Register($src1$$reg),
14577             (uint64_t)($src2$$constant));
14578   %}
14579 
14580   ins_pipe(ialu_reg_imm);
14581 %}
14582 
14583 // Or Instructions
14584 
14585 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14586   match(Set dst (OrI src1 src2));
14587 
14588   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14589 
14590   ins_cost(INSN_COST);
14591   ins_encode %{
14592     __ orrw(as_Register($dst$$reg),
14593             as_Register($src1$$reg),
14594             as_Register($src2$$reg));
14595   %}
14596 
14597   ins_pipe(ialu_reg_reg);
14598 %}
14599 
14600 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14601   match(Set dst (OrI src1 src2));
14602 
14603   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14604 
14605   ins_cost(INSN_COST);
14606   ins_encode %{
14607     __ orrw(as_Register($dst$$reg),
14608             as_Register($src1$$reg),
14609             (uint64_t)($src2$$constant));
14610   %}
14611 
14612   ins_pipe(ialu_reg_imm);
14613 %}
14614 
14615 // Xor Instructions
14616 
14617 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14618   match(Set dst (XorI src1 src2));
14619 
14620   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14621 
14622   ins_cost(INSN_COST);
14623   ins_encode %{
14624     __ eorw(as_Register($dst$$reg),
14625             as_Register($src1$$reg),
14626             as_Register($src2$$reg));
14627   %}
14628 
14629   ins_pipe(ialu_reg_reg);
14630 %}
14631 
14632 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14633   match(Set dst (XorI src1 src2));
14634 
14635   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14636 
14637   ins_cost(INSN_COST);
14638   ins_encode %{
14639     __ eorw(as_Register($dst$$reg),
14640             as_Register($src1$$reg),
14641             (uint64_t)($src2$$constant));
14642   %}
14643 
14644   ins_pipe(ialu_reg_imm);
14645 %}
14646 
14647 // Long Logical Instructions
14648 // TODO
14649 
14650 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14651   match(Set dst (AndL src1 src2));
14652 
14653   format %{ "and  $dst, $src1, $src2\t# int" %}
14654 
14655   ins_cost(INSN_COST);
14656   ins_encode %{
14657     __ andr(as_Register($dst$$reg),
14658             as_Register($src1$$reg),
14659             as_Register($src2$$reg));
14660   %}
14661 
14662   ins_pipe(ialu_reg_reg);
14663 %}
14664 
14665 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14666   match(Set dst (AndL src1 src2));
14667 
14668   format %{ "and  $dst, $src1, $src2\t# int" %}
14669 
14670   ins_cost(INSN_COST);
14671   ins_encode %{
14672     __ andr(as_Register($dst$$reg),
14673             as_Register($src1$$reg),
14674             (uint64_t)($src2$$constant));
14675   %}
14676 
14677   ins_pipe(ialu_reg_imm);
14678 %}
14679 
14680 // Or Instructions
14681 
14682 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14683   match(Set dst (OrL src1 src2));
14684 
14685   format %{ "orr  $dst, $src1, $src2\t# int" %}
14686 
14687   ins_cost(INSN_COST);
14688   ins_encode %{
14689     __ orr(as_Register($dst$$reg),
14690            as_Register($src1$$reg),
14691            as_Register($src2$$reg));
14692   %}
14693 
14694   ins_pipe(ialu_reg_reg);
14695 %}
14696 
14697 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14698   match(Set dst (OrL src1 src2));
14699 
14700   format %{ "orr  $dst, $src1, $src2\t# int" %}
14701 
14702   ins_cost(INSN_COST);
14703   ins_encode %{
14704     __ orr(as_Register($dst$$reg),
14705            as_Register($src1$$reg),
14706            (uint64_t)($src2$$constant));
14707   %}
14708 
14709   ins_pipe(ialu_reg_imm);
14710 %}
14711 
14712 // Xor Instructions
14713 
14714 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14715   match(Set dst (XorL src1 src2));
14716 
14717   format %{ "eor  $dst, $src1, $src2\t# int" %}
14718 
14719   ins_cost(INSN_COST);
14720   ins_encode %{
14721     __ eor(as_Register($dst$$reg),
14722            as_Register($src1$$reg),
14723            as_Register($src2$$reg));
14724   %}
14725 
14726   ins_pipe(ialu_reg_reg);
14727 %}
14728 
14729 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14730   match(Set dst (XorL src1 src2));
14731 
14732   ins_cost(INSN_COST);
14733   format %{ "eor  $dst, $src1, $src2\t# int" %}
14734 
14735   ins_encode %{
14736     __ eor(as_Register($dst$$reg),
14737            as_Register($src1$$reg),
14738            (uint64_t)($src2$$constant));
14739   %}
14740 
14741   ins_pipe(ialu_reg_imm);
14742 %}
14743 
14744 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14745 %{
14746   match(Set dst (ConvI2L src));
14747 
14748   ins_cost(INSN_COST);
14749   format %{ "sxtw  $dst, $src\t# i2l" %}
14750   ins_encode %{
14751     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14752   %}
14753   ins_pipe(ialu_reg_shift);
14754 %}
14755 
14756 // this pattern occurs in bigmath arithmetic
14757 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14758 %{
14759   match(Set dst (AndL (ConvI2L src) mask));
14760 
14761   ins_cost(INSN_COST);
14762   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14763   ins_encode %{
14764     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14765   %}
14766 
14767   ins_pipe(ialu_reg_shift);
14768 %}
14769 
14770 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14771   match(Set dst (ConvL2I src));
14772 
14773   ins_cost(INSN_COST);
14774   format %{ "movw  $dst, $src \t// l2i" %}
14775 
14776   ins_encode %{
14777     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14778   %}
14779 
14780   ins_pipe(ialu_reg);
14781 %}
14782 
14783 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14784 %{
14785   match(Set dst (Conv2B src));
14786   effect(KILL cr);
14787 
14788   format %{
14789     "cmpw $src, zr\n\t"
14790     "cset $dst, ne"
14791   %}
14792 
14793   ins_encode %{
14794     __ cmpw(as_Register($src$$reg), zr);
14795     __ cset(as_Register($dst$$reg), Assembler::NE);
14796   %}
14797 
14798   ins_pipe(ialu_reg);
14799 %}
14800 
14801 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14802 %{
14803   match(Set dst (Conv2B src));
14804   effect(KILL cr);
14805 
14806   format %{
14807     "cmp  $src, zr\n\t"
14808     "cset $dst, ne"
14809   %}
14810 
14811   ins_encode %{
14812     __ cmp(as_Register($src$$reg), zr);
14813     __ cset(as_Register($dst$$reg), Assembler::NE);
14814   %}
14815 
14816   ins_pipe(ialu_reg);
14817 %}
14818 
14819 instruct convD2F_reg(vRegF dst, vRegD src) %{
14820   match(Set dst (ConvD2F src));
14821 
14822   ins_cost(INSN_COST * 5);
14823   format %{ "fcvtd  $dst, $src \t// d2f" %}
14824 
14825   ins_encode %{
14826     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14827   %}
14828 
14829   ins_pipe(fp_d2f);
14830 %}
14831 
14832 instruct convF2D_reg(vRegD dst, vRegF src) %{
14833   match(Set dst (ConvF2D src));
14834 
14835   ins_cost(INSN_COST * 5);
14836   format %{ "fcvts  $dst, $src \t// f2d" %}
14837 
14838   ins_encode %{
14839     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14840   %}
14841 
14842   ins_pipe(fp_f2d);
14843 %}
14844 
14845 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14846   match(Set dst (ConvF2I src));
14847 
14848   ins_cost(INSN_COST * 5);
14849   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14850 
14851   ins_encode %{
14852     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14853   %}
14854 
14855   ins_pipe(fp_f2i);
14856 %}
14857 
14858 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14859   match(Set dst (ConvF2L src));
14860 
14861   ins_cost(INSN_COST * 5);
14862   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14863 
14864   ins_encode %{
14865     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14866   %}
14867 
14868   ins_pipe(fp_f2l);
14869 %}
14870 
14871 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14872   match(Set dst (ConvI2F src));
14873 
14874   ins_cost(INSN_COST * 5);
14875   format %{ "scvtfws  $dst, $src \t// i2f" %}
14876 
14877   ins_encode %{
14878     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14879   %}
14880 
14881   ins_pipe(fp_i2f);
14882 %}
14883 
14884 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14885   match(Set dst (ConvL2F src));
14886 
14887   ins_cost(INSN_COST * 5);
14888   format %{ "scvtfs  $dst, $src \t// l2f" %}
14889 
14890   ins_encode %{
14891     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14892   %}
14893 
14894   ins_pipe(fp_l2f);
14895 %}
14896 
14897 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14898   match(Set dst (ConvD2I src));
14899 
14900   ins_cost(INSN_COST * 5);
14901   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14902 
14903   ins_encode %{
14904     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14905   %}
14906 
14907   ins_pipe(fp_d2i);
14908 %}
14909 
14910 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14911   match(Set dst (ConvD2L src));
14912 
14913   ins_cost(INSN_COST * 5);
14914   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14915 
14916   ins_encode %{
14917     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14918   %}
14919 
14920   ins_pipe(fp_d2l);
14921 %}
14922 
14923 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14924   match(Set dst (ConvI2D src));
14925 
14926   ins_cost(INSN_COST * 5);
14927   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14928 
14929   ins_encode %{
14930     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14931   %}
14932 
14933   ins_pipe(fp_i2d);
14934 %}
14935 
14936 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14937   match(Set dst (ConvL2D src));
14938 
14939   ins_cost(INSN_COST * 5);
14940   format %{ "scvtfd  $dst, $src \t// l2d" %}
14941 
14942   ins_encode %{
14943     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14944   %}
14945 
14946   ins_pipe(fp_l2d);
14947 %}
14948 
14949 // stack <-> reg and reg <-> reg shuffles with no conversion
14950 
14951 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14952 
14953   match(Set dst (MoveF2I src));
14954 
14955   effect(DEF dst, USE src);
14956 
14957   ins_cost(4 * INSN_COST);
14958 
14959   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14960 
14961   ins_encode %{
14962     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14963   %}
14964 
14965   ins_pipe(iload_reg_reg);
14966 
14967 %}
14968 
14969 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14970 
14971   match(Set dst (MoveI2F src));
14972 
14973   effect(DEF dst, USE src);
14974 
14975   ins_cost(4 * INSN_COST);
14976 
14977   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14978 
14979   ins_encode %{
14980     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14981   %}
14982 
14983   ins_pipe(pipe_class_memory);
14984 
14985 %}
14986 
14987 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14988 
14989   match(Set dst (MoveD2L src));
14990 
14991   effect(DEF dst, USE src);
14992 
14993   ins_cost(4 * INSN_COST);
14994 
14995   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14996 
14997   ins_encode %{
14998     __ ldr($dst$$Register, Address(sp, $src$$disp));
14999   %}
15000 
15001   ins_pipe(iload_reg_reg);
15002 
15003 %}
15004 
15005 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15006 
15007   match(Set dst (MoveL2D src));
15008 
15009   effect(DEF dst, USE src);
15010 
15011   ins_cost(4 * INSN_COST);
15012 
15013   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15014 
15015   ins_encode %{
15016     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15017   %}
15018 
15019   ins_pipe(pipe_class_memory);
15020 
15021 %}
15022 
15023 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15024 
15025   match(Set dst (MoveF2I src));
15026 
15027   effect(DEF dst, USE src);
15028 
15029   ins_cost(INSN_COST);
15030 
15031   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15032 
15033   ins_encode %{
15034     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15035   %}
15036 
15037   ins_pipe(pipe_class_memory);
15038 
15039 %}
15040 
15041 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15042 
15043   match(Set dst (MoveI2F src));
15044 
15045   effect(DEF dst, USE src);
15046 
15047   ins_cost(INSN_COST);
15048 
15049   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15050 
15051   ins_encode %{
15052     __ strw($src$$Register, Address(sp, $dst$$disp));
15053   %}
15054 
15055   ins_pipe(istore_reg_reg);
15056 
15057 %}
15058 
15059 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15060 
15061   match(Set dst (MoveD2L src));
15062 
15063   effect(DEF dst, USE src);
15064 
15065   ins_cost(INSN_COST);
15066 
15067   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15068 
15069   ins_encode %{
15070     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15071   %}
15072 
15073   ins_pipe(pipe_class_memory);
15074 
15075 %}
15076 
15077 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15078 
15079   match(Set dst (MoveL2D src));
15080 
15081   effect(DEF dst, USE src);
15082 
15083   ins_cost(INSN_COST);
15084 
15085   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15086 
15087   ins_encode %{
15088     __ str($src$$Register, Address(sp, $dst$$disp));
15089   %}
15090 
15091   ins_pipe(istore_reg_reg);
15092 
15093 %}
15094 
15095 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15096 
15097   match(Set dst (MoveF2I src));
15098 
15099   effect(DEF dst, USE src);
15100 
15101   ins_cost(INSN_COST);
15102 
15103   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15104 
15105   ins_encode %{
15106     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15107   %}
15108 
15109   ins_pipe(fp_f2i);
15110 
15111 %}
15112 
15113 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15114 
15115   match(Set dst (MoveI2F src));
15116 
15117   effect(DEF dst, USE src);
15118 
15119   ins_cost(INSN_COST);
15120 
15121   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15122 
15123   ins_encode %{
15124     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15125   %}
15126 
15127   ins_pipe(fp_i2f);
15128 
15129 %}
15130 
15131 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15132 
15133   match(Set dst (MoveD2L src));
15134 
15135   effect(DEF dst, USE src);
15136 
15137   ins_cost(INSN_COST);
15138 
15139   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15140 
15141   ins_encode %{
15142     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15143   %}
15144 
15145   ins_pipe(fp_d2l);
15146 
15147 %}
15148 
15149 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15150 
15151   match(Set dst (MoveL2D src));
15152 
15153   effect(DEF dst, USE src);
15154 
15155   ins_cost(INSN_COST);
15156 
15157   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15158 
15159   ins_encode %{
15160     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15161   %}
15162 
15163   ins_pipe(fp_l2d);
15164 
15165 %}
15166 
15167 // ============================================================================
15168 // clearing of an array
15169 
15170 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15171 %{
15172   match(Set dummy (ClearArray cnt base));
15173   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15174 
15175   ins_cost(4 * INSN_COST);
15176   format %{ "ClearArray $cnt, $base" %}
15177 
15178   ins_encode %{
15179     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15180     if (tpc == NULL) {
15181       ciEnv::current()->record_failure("CodeCache is full");
15182       return;
15183     }
15184   %}
15185 
15186   ins_pipe(pipe_class_memory);
15187 %}
15188 
15189 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15190 %{
15191   predicate((uint64_t)n->in(2)->get_long()
15192             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15193   match(Set dummy (ClearArray cnt base));
15194   effect(TEMP temp, USE_KILL base, KILL cr);
15195 
15196   ins_cost(4 * INSN_COST);
15197   format %{ "ClearArray $cnt, $base" %}
15198 
15199   ins_encode %{
15200     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15201   %}
15202 
15203   ins_pipe(pipe_class_memory);
15204 %}
15205 
15206 // ============================================================================
15207 // Overflow Math Instructions
15208 
15209 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15210 %{
15211   match(Set cr (OverflowAddI op1 op2));
15212 
15213   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15214   ins_cost(INSN_COST);
15215   ins_encode %{
15216     __ cmnw($op1$$Register, $op2$$Register);
15217   %}
15218 
15219   ins_pipe(icmp_reg_reg);
15220 %}
15221 
15222 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15223 %{
15224   match(Set cr (OverflowAddI op1 op2));
15225 
15226   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15227   ins_cost(INSN_COST);
15228   ins_encode %{
15229     __ cmnw($op1$$Register, $op2$$constant);
15230   %}
15231 
15232   ins_pipe(icmp_reg_imm);
15233 %}
15234 
15235 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15236 %{
15237   match(Set cr (OverflowAddL op1 op2));
15238 
15239   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15240   ins_cost(INSN_COST);
15241   ins_encode %{
15242     __ cmn($op1$$Register, $op2$$Register);
15243   %}
15244 
15245   ins_pipe(icmp_reg_reg);
15246 %}
15247 
15248 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15249 %{
15250   match(Set cr (OverflowAddL op1 op2));
15251 
15252   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15253   ins_cost(INSN_COST);
15254   ins_encode %{
15255     __ cmn($op1$$Register, $op2$$constant);
15256   %}
15257 
15258   ins_pipe(icmp_reg_imm);
15259 %}
15260 
15261 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15262 %{
15263   match(Set cr (OverflowSubI op1 op2));
15264 
15265   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15266   ins_cost(INSN_COST);
15267   ins_encode %{
15268     __ cmpw($op1$$Register, $op2$$Register);
15269   %}
15270 
15271   ins_pipe(icmp_reg_reg);
15272 %}
15273 
15274 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15275 %{
15276   match(Set cr (OverflowSubI op1 op2));
15277 
15278   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15279   ins_cost(INSN_COST);
15280   ins_encode %{
15281     __ cmpw($op1$$Register, $op2$$constant);
15282   %}
15283 
15284   ins_pipe(icmp_reg_imm);
15285 %}
15286 
15287 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15288 %{
15289   match(Set cr (OverflowSubL op1 op2));
15290 
15291   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15292   ins_cost(INSN_COST);
15293   ins_encode %{
15294     __ cmp($op1$$Register, $op2$$Register);
15295   %}
15296 
15297   ins_pipe(icmp_reg_reg);
15298 %}
15299 
15300 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15301 %{
15302   match(Set cr (OverflowSubL op1 op2));
15303 
15304   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15305   ins_cost(INSN_COST);
15306   ins_encode %{
15307     __ subs(zr, $op1$$Register, $op2$$constant);
15308   %}
15309 
15310   ins_pipe(icmp_reg_imm);
15311 %}
15312 
15313 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15314 %{
15315   match(Set cr (OverflowSubI zero op1));
15316 
15317   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15318   ins_cost(INSN_COST);
15319   ins_encode %{
15320     __ cmpw(zr, $op1$$Register);
15321   %}
15322 
15323   ins_pipe(icmp_reg_imm);
15324 %}
15325 
15326 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15327 %{
15328   match(Set cr (OverflowSubL zero op1));
15329 
15330   format %{ "cmp   zr, $op1\t# overflow check long" %}
15331   ins_cost(INSN_COST);
15332   ins_encode %{
15333     __ cmp(zr, $op1$$Register);
15334   %}
15335 
15336   ins_pipe(icmp_reg_imm);
15337 %}
15338 
15339 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15340 %{
15341   match(Set cr (OverflowMulI op1 op2));
15342 
15343   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15344             "cmp   rscratch1, rscratch1, sxtw\n\t"
15345             "movw  rscratch1, #0x80000000\n\t"
15346             "cselw rscratch1, rscratch1, zr, NE\n\t"
15347             "cmpw  rscratch1, #1" %}
15348   ins_cost(5 * INSN_COST);
15349   ins_encode %{
15350     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15351     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15352     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15353     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15354     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15355   %}
15356 
15357   ins_pipe(pipe_slow);
15358 %}
15359 
15360 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15361 %{
15362   match(If cmp (OverflowMulI op1 op2));
15363   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15364             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15365   effect(USE labl, KILL cr);
15366 
15367   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15368             "cmp   rscratch1, rscratch1, sxtw\n\t"
15369             "b$cmp   $labl" %}
15370   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15371   ins_encode %{
15372     Label* L = $labl$$label;
15373     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15374     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15375     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15376     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15377   %}
15378 
15379   ins_pipe(pipe_serial);
15380 %}
15381 
15382 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15383 %{
15384   match(Set cr (OverflowMulL op1 op2));
15385 
15386   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15387             "smulh rscratch2, $op1, $op2\n\t"
15388             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15389             "movw  rscratch1, #0x80000000\n\t"
15390             "cselw rscratch1, rscratch1, zr, NE\n\t"
15391             "cmpw  rscratch1, #1" %}
15392   ins_cost(6 * INSN_COST);
15393   ins_encode %{
15394     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15395     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15396     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15397     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15398     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15399     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15400   %}
15401 
15402   ins_pipe(pipe_slow);
15403 %}
15404 
15405 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15406 %{
15407   match(If cmp (OverflowMulL op1 op2));
15408   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15409             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15410   effect(USE labl, KILL cr);
15411 
15412   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15413             "smulh rscratch2, $op1, $op2\n\t"
15414             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15415             "b$cmp $labl" %}
15416   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15417   ins_encode %{
15418     Label* L = $labl$$label;
15419     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15420     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15421     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15422     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15423     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15424   %}
15425 
15426   ins_pipe(pipe_serial);
15427 %}
15428 
15429 // ============================================================================
15430 // Compare Instructions
15431 
15432 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15433 %{
15434   match(Set cr (CmpI op1 op2));
15435 
15436   effect(DEF cr, USE op1, USE op2);
15437 
15438   ins_cost(INSN_COST);
15439   format %{ "cmpw  $op1, $op2" %}
15440 
15441   ins_encode(aarch64_enc_cmpw(op1, op2));
15442 
15443   ins_pipe(icmp_reg_reg);
15444 %}
15445 
15446 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15447 %{
15448   match(Set cr (CmpI op1 zero));
15449 
15450   effect(DEF cr, USE op1);
15451 
15452   ins_cost(INSN_COST);
15453   format %{ "cmpw $op1, 0" %}
15454 
15455   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15456 
15457   ins_pipe(icmp_reg_imm);
15458 %}
15459 
15460 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15461 %{
15462   match(Set cr (CmpI op1 op2));
15463 
15464   effect(DEF cr, USE op1);
15465 
15466   ins_cost(INSN_COST);
15467   format %{ "cmpw  $op1, $op2" %}
15468 
15469   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15470 
15471   ins_pipe(icmp_reg_imm);
15472 %}
15473 
15474 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15475 %{
15476   match(Set cr (CmpI op1 op2));
15477 
15478   effect(DEF cr, USE op1);
15479 
15480   ins_cost(INSN_COST * 2);
15481   format %{ "cmpw  $op1, $op2" %}
15482 
15483   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15484 
15485   ins_pipe(icmp_reg_imm);
15486 %}
15487 
15488 // Unsigned compare Instructions; really, same as signed compare
15489 // except it should only be used to feed an If or a CMovI which takes a
15490 // cmpOpU.
15491 
15492 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15493 %{
15494   match(Set cr (CmpU op1 op2));
15495 
15496   effect(DEF cr, USE op1, USE op2);
15497 
15498   ins_cost(INSN_COST);
15499   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15500 
15501   ins_encode(aarch64_enc_cmpw(op1, op2));
15502 
15503   ins_pipe(icmp_reg_reg);
15504 %}
15505 
15506 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15507 %{
15508   match(Set cr (CmpU op1 zero));
15509 
15510   effect(DEF cr, USE op1);
15511 
15512   ins_cost(INSN_COST);
15513   format %{ "cmpw $op1, #0\t# unsigned" %}
15514 
15515   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15516 
15517   ins_pipe(icmp_reg_imm);
15518 %}
15519 
15520 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15521 %{
15522   match(Set cr (CmpU op1 op2));
15523 
15524   effect(DEF cr, USE op1);
15525 
15526   ins_cost(INSN_COST);
15527   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15528 
15529   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15530 
15531   ins_pipe(icmp_reg_imm);
15532 %}
15533 
15534 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15535 %{
15536   match(Set cr (CmpU op1 op2));
15537 
15538   effect(DEF cr, USE op1);
15539 
15540   ins_cost(INSN_COST * 2);
15541   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15542 
15543   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15544 
15545   ins_pipe(icmp_reg_imm);
15546 %}
15547 
15548 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15549 %{
15550   match(Set cr (CmpL op1 op2));
15551 
15552   effect(DEF cr, USE op1, USE op2);
15553 
15554   ins_cost(INSN_COST);
15555   format %{ "cmp  $op1, $op2" %}
15556 
15557   ins_encode(aarch64_enc_cmp(op1, op2));
15558 
15559   ins_pipe(icmp_reg_reg);
15560 %}
15561 
15562 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15563 %{
15564   match(Set cr (CmpL op1 zero));
15565 
15566   effect(DEF cr, USE op1);
15567 
15568   ins_cost(INSN_COST);
15569   format %{ "tst  $op1" %}
15570 
15571   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15572 
15573   ins_pipe(icmp_reg_imm);
15574 %}
15575 
15576 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15577 %{
15578   match(Set cr (CmpL op1 op2));
15579 
15580   effect(DEF cr, USE op1);
15581 
15582   ins_cost(INSN_COST);
15583   format %{ "cmp  $op1, $op2" %}
15584 
15585   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15586 
15587   ins_pipe(icmp_reg_imm);
15588 %}
15589 
15590 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15591 %{
15592   match(Set cr (CmpL op1 op2));
15593 
15594   effect(DEF cr, USE op1);
15595 
15596   ins_cost(INSN_COST * 2);
15597   format %{ "cmp  $op1, $op2" %}
15598 
15599   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15600 
15601   ins_pipe(icmp_reg_imm);
15602 %}
15603 
15604 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15605 %{
15606   match(Set cr (CmpUL op1 op2));
15607 
15608   effect(DEF cr, USE op1, USE op2);
15609 
15610   ins_cost(INSN_COST);
15611   format %{ "cmp  $op1, $op2" %}
15612 
15613   ins_encode(aarch64_enc_cmp(op1, op2));
15614 
15615   ins_pipe(icmp_reg_reg);
15616 %}
15617 
15618 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15619 %{
15620   match(Set cr (CmpUL op1 zero));
15621 
15622   effect(DEF cr, USE op1);
15623 
15624   ins_cost(INSN_COST);
15625   format %{ "tst  $op1" %}
15626 
15627   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15628 
15629   ins_pipe(icmp_reg_imm);
15630 %}
15631 
15632 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15633 %{
15634   match(Set cr (CmpUL op1 op2));
15635 
15636   effect(DEF cr, USE op1);
15637 
15638   ins_cost(INSN_COST);
15639   format %{ "cmp  $op1, $op2" %}
15640 
15641   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15642 
15643   ins_pipe(icmp_reg_imm);
15644 %}
15645 
15646 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15647 %{
15648   match(Set cr (CmpUL op1 op2));
15649 
15650   effect(DEF cr, USE op1);
15651 
15652   ins_cost(INSN_COST * 2);
15653   format %{ "cmp  $op1, $op2" %}
15654 
15655   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15656 
15657   ins_pipe(icmp_reg_imm);
15658 %}
15659 
15660 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15661 %{
15662   match(Set cr (CmpP op1 op2));
15663 
15664   effect(DEF cr, USE op1, USE op2);
15665 
15666   ins_cost(INSN_COST);
15667   format %{ "cmp  $op1, $op2\t // ptr" %}
15668 
15669   ins_encode(aarch64_enc_cmpp(op1, op2));
15670 
15671   ins_pipe(icmp_reg_reg);
15672 %}
15673 
15674 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15675 %{
15676   match(Set cr (CmpN op1 op2));
15677 
15678   effect(DEF cr, USE op1, USE op2);
15679 
15680   ins_cost(INSN_COST);
15681   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15682 
15683   ins_encode(aarch64_enc_cmpn(op1, op2));
15684 
15685   ins_pipe(icmp_reg_reg);
15686 %}
15687 
15688 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15689 %{
15690   match(Set cr (CmpP op1 zero));
15691 
15692   effect(DEF cr, USE op1, USE zero);
15693 
15694   ins_cost(INSN_COST);
15695   format %{ "cmp  $op1, 0\t // ptr" %}
15696 
15697   ins_encode(aarch64_enc_testp(op1));
15698 
15699   ins_pipe(icmp_reg_imm);
15700 %}
15701 
15702 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15703 %{
15704   match(Set cr (CmpN op1 zero));
15705 
15706   effect(DEF cr, USE op1, USE zero);
15707 
15708   ins_cost(INSN_COST);
15709   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15710 
15711   ins_encode(aarch64_enc_testn(op1));
15712 
15713   ins_pipe(icmp_reg_imm);
15714 %}
15715 
15716 // FP comparisons
15717 //
15718 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15719 // using normal cmpOp. See declaration of rFlagsReg for details.
15720 
15721 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15722 %{
15723   match(Set cr (CmpF src1 src2));
15724 
15725   ins_cost(3 * INSN_COST);
15726   format %{ "fcmps $src1, $src2" %}
15727 
15728   ins_encode %{
15729     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15730   %}
15731 
15732   ins_pipe(pipe_class_compare);
15733 %}
15734 
15735 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15736 %{
15737   match(Set cr (CmpF src1 src2));
15738 
15739   ins_cost(3 * INSN_COST);
15740   format %{ "fcmps $src1, 0.0" %}
15741 
15742   ins_encode %{
15743     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15744   %}
15745 
15746   ins_pipe(pipe_class_compare);
15747 %}
15748 // FROM HERE
15749 
15750 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15751 %{
15752   match(Set cr (CmpD src1 src2));
15753 
15754   ins_cost(3 * INSN_COST);
15755   format %{ "fcmpd $src1, $src2" %}
15756 
15757   ins_encode %{
15758     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15759   %}
15760 
15761   ins_pipe(pipe_class_compare);
15762 %}
15763 
15764 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15765 %{
15766   match(Set cr (CmpD src1 src2));
15767 
15768   ins_cost(3 * INSN_COST);
15769   format %{ "fcmpd $src1, 0.0" %}
15770 
15771   ins_encode %{
15772     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15773   %}
15774 
15775   ins_pipe(pipe_class_compare);
15776 %}
15777 
15778 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15779 %{
15780   match(Set dst (CmpF3 src1 src2));
15781   effect(KILL cr);
15782 
15783   ins_cost(5 * INSN_COST);
15784   format %{ "fcmps $src1, $src2\n\t"
15785             "csinvw($dst, zr, zr, eq\n\t"
15786             "csnegw($dst, $dst, $dst, lt)"
15787   %}
15788 
15789   ins_encode %{
15790     Label done;
15791     FloatRegister s1 = as_FloatRegister($src1$$reg);
15792     FloatRegister s2 = as_FloatRegister($src2$$reg);
15793     Register d = as_Register($dst$$reg);
15794     __ fcmps(s1, s2);
15795     // installs 0 if EQ else -1
15796     __ csinvw(d, zr, zr, Assembler::EQ);
15797     // keeps -1 if less or unordered else installs 1
15798     __ csnegw(d, d, d, Assembler::LT);
15799     __ bind(done);
15800   %}
15801 
15802   ins_pipe(pipe_class_default);
15803 
15804 %}
15805 
15806 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15807 %{
15808   match(Set dst (CmpD3 src1 src2));
15809   effect(KILL cr);
15810 
15811   ins_cost(5 * INSN_COST);
15812   format %{ "fcmpd $src1, $src2\n\t"
15813             "csinvw($dst, zr, zr, eq\n\t"
15814             "csnegw($dst, $dst, $dst, lt)"
15815   %}
15816 
15817   ins_encode %{
15818     Label done;
15819     FloatRegister s1 = as_FloatRegister($src1$$reg);
15820     FloatRegister s2 = as_FloatRegister($src2$$reg);
15821     Register d = as_Register($dst$$reg);
15822     __ fcmpd(s1, s2);
15823     // installs 0 if EQ else -1
15824     __ csinvw(d, zr, zr, Assembler::EQ);
15825     // keeps -1 if less or unordered else installs 1
15826     __ csnegw(d, d, d, Assembler::LT);
15827     __ bind(done);
15828   %}
15829   ins_pipe(pipe_class_default);
15830 
15831 %}
15832 
15833 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15834 %{
15835   match(Set dst (CmpF3 src1 zero));
15836   effect(KILL cr);
15837 
15838   ins_cost(5 * INSN_COST);
15839   format %{ "fcmps $src1, 0.0\n\t"
15840             "csinvw($dst, zr, zr, eq\n\t"
15841             "csnegw($dst, $dst, $dst, lt)"
15842   %}
15843 
15844   ins_encode %{
15845     Label done;
15846     FloatRegister s1 = as_FloatRegister($src1$$reg);
15847     Register d = as_Register($dst$$reg);
15848     __ fcmps(s1, 0.0);
15849     // installs 0 if EQ else -1
15850     __ csinvw(d, zr, zr, Assembler::EQ);
15851     // keeps -1 if less or unordered else installs 1
15852     __ csnegw(d, d, d, Assembler::LT);
15853     __ bind(done);
15854   %}
15855 
15856   ins_pipe(pipe_class_default);
15857 
15858 %}
15859 
15860 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15861 %{
15862   match(Set dst (CmpD3 src1 zero));
15863   effect(KILL cr);
15864 
15865   ins_cost(5 * INSN_COST);
15866   format %{ "fcmpd $src1, 0.0\n\t"
15867             "csinvw($dst, zr, zr, eq\n\t"
15868             "csnegw($dst, $dst, $dst, lt)"
15869   %}
15870 
15871   ins_encode %{
15872     Label done;
15873     FloatRegister s1 = as_FloatRegister($src1$$reg);
15874     Register d = as_Register($dst$$reg);
15875     __ fcmpd(s1, 0.0);
15876     // installs 0 if EQ else -1
15877     __ csinvw(d, zr, zr, Assembler::EQ);
15878     // keeps -1 if less or unordered else installs 1
15879     __ csnegw(d, d, d, Assembler::LT);
15880     __ bind(done);
15881   %}
15882   ins_pipe(pipe_class_default);
15883 
15884 %}
15885 
15886 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15887 %{
15888   match(Set dst (CmpLTMask p q));
15889   effect(KILL cr);
15890 
15891   ins_cost(3 * INSN_COST);
15892 
15893   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15894             "csetw $dst, lt\n\t"
15895             "subw $dst, zr, $dst"
15896   %}
15897 
15898   ins_encode %{
15899     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15900     __ csetw(as_Register($dst$$reg), Assembler::LT);
15901     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15902   %}
15903 
15904   ins_pipe(ialu_reg_reg);
15905 %}
15906 
15907 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15908 %{
15909   match(Set dst (CmpLTMask src zero));
15910   effect(KILL cr);
15911 
15912   ins_cost(INSN_COST);
15913 
15914   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15915 
15916   ins_encode %{
15917     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15918   %}
15919 
15920   ins_pipe(ialu_reg_shift);
15921 %}
15922 
15923 // ============================================================================
15924 // Max and Min
15925 
15926 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15927 %{
15928   effect( DEF dst, USE src1, USE src2, USE cr );
15929 
15930   ins_cost(INSN_COST * 2);
15931   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15932 
15933   ins_encode %{
15934     __ cselw(as_Register($dst$$reg),
15935              as_Register($src1$$reg),
15936              as_Register($src2$$reg),
15937              Assembler::LT);
15938   %}
15939 
15940   ins_pipe(icond_reg_reg);
15941 %}
15942 
15943 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15944 %{
15945   match(Set dst (MinI src1 src2));
15946   ins_cost(INSN_COST * 3);
15947 
15948   expand %{
15949     rFlagsReg cr;
15950     compI_reg_reg(cr, src1, src2);
15951     cmovI_reg_reg_lt(dst, src1, src2, cr);
15952   %}
15953 
15954 %}
15955 // FROM HERE
15956 
15957 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15958 %{
15959   effect( DEF dst, USE src1, USE src2, USE cr );
15960 
15961   ins_cost(INSN_COST * 2);
15962   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15963 
15964   ins_encode %{
15965     __ cselw(as_Register($dst$$reg),
15966              as_Register($src1$$reg),
15967              as_Register($src2$$reg),
15968              Assembler::GT);
15969   %}
15970 
15971   ins_pipe(icond_reg_reg);
15972 %}
15973 
15974 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15975 %{
15976   match(Set dst (MaxI src1 src2));
15977   ins_cost(INSN_COST * 3);
15978   expand %{
15979     rFlagsReg cr;
15980     compI_reg_reg(cr, src1, src2);
15981     cmovI_reg_reg_gt(dst, src1, src2, cr);
15982   %}
15983 %}
15984 
15985 // ============================================================================
15986 // Branch Instructions
15987 
15988 // Direct Branch.
15989 instruct branch(label lbl)
15990 %{
15991   match(Goto);
15992 
15993   effect(USE lbl);
15994 
15995   ins_cost(BRANCH_COST);
15996   format %{ "b  $lbl" %}
15997 
15998   ins_encode(aarch64_enc_b(lbl));
15999 
16000   ins_pipe(pipe_branch);
16001 %}
16002 
16003 // Conditional Near Branch
16004 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16005 %{
16006   // Same match rule as `branchConFar'.
16007   match(If cmp cr);
16008 
16009   effect(USE lbl);
16010 
16011   ins_cost(BRANCH_COST);
16012   // If set to 1 this indicates that the current instruction is a
16013   // short variant of a long branch. This avoids using this
16014   // instruction in first-pass matching. It will then only be used in
16015   // the `Shorten_branches' pass.
16016   // ins_short_branch(1);
16017   format %{ "b$cmp  $lbl" %}
16018 
16019   ins_encode(aarch64_enc_br_con(cmp, lbl));
16020 
16021   ins_pipe(pipe_branch_cond);
16022 %}
16023 
16024 // Conditional Near Branch Unsigned
16025 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16026 %{
16027   // Same match rule as `branchConFar'.
16028   match(If cmp cr);
16029 
16030   effect(USE lbl);
16031 
16032   ins_cost(BRANCH_COST);
16033   // If set to 1 this indicates that the current instruction is a
16034   // short variant of a long branch. This avoids using this
16035   // instruction in first-pass matching. It will then only be used in
16036   // the `Shorten_branches' pass.
16037   // ins_short_branch(1);
16038   format %{ "b$cmp  $lbl\t# unsigned" %}
16039 
16040   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16041 
16042   ins_pipe(pipe_branch_cond);
16043 %}
16044 
16045 // Make use of CBZ and CBNZ.  These instructions, as well as being
16046 // shorter than (cmp; branch), have the additional benefit of not
16047 // killing the flags.
16048 
16049 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16050   match(If cmp (CmpI op1 op2));
16051   effect(USE labl);
16052 
16053   ins_cost(BRANCH_COST);
16054   format %{ "cbw$cmp   $op1, $labl" %}
16055   ins_encode %{
16056     Label* L = $labl$$label;
16057     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16058     if (cond == Assembler::EQ)
16059       __ cbzw($op1$$Register, *L);
16060     else
16061       __ cbnzw($op1$$Register, *L);
16062   %}
16063   ins_pipe(pipe_cmp_branch);
16064 %}
16065 
16066 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16067   match(If cmp (CmpL op1 op2));
16068   effect(USE labl);
16069 
16070   ins_cost(BRANCH_COST);
16071   format %{ "cb$cmp   $op1, $labl" %}
16072   ins_encode %{
16073     Label* L = $labl$$label;
16074     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16075     if (cond == Assembler::EQ)
16076       __ cbz($op1$$Register, *L);
16077     else
16078       __ cbnz($op1$$Register, *L);
16079   %}
16080   ins_pipe(pipe_cmp_branch);
16081 %}
16082 
16083 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16084   match(If cmp (CmpP op1 op2));
16085   effect(USE labl);
16086 
16087   ins_cost(BRANCH_COST);
16088   format %{ "cb$cmp   $op1, $labl" %}
16089   ins_encode %{
16090     Label* L = $labl$$label;
16091     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16092     if (cond == Assembler::EQ)
16093       __ cbz($op1$$Register, *L);
16094     else
16095       __ cbnz($op1$$Register, *L);
16096   %}
16097   ins_pipe(pipe_cmp_branch);
16098 %}
16099 
16100 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16101   match(If cmp (CmpN op1 op2));
16102   effect(USE labl);
16103 
16104   ins_cost(BRANCH_COST);
16105   format %{ "cbw$cmp   $op1, $labl" %}
16106   ins_encode %{
16107     Label* L = $labl$$label;
16108     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16109     if (cond == Assembler::EQ)
16110       __ cbzw($op1$$Register, *L);
16111     else
16112       __ cbnzw($op1$$Register, *L);
16113   %}
16114   ins_pipe(pipe_cmp_branch);
16115 %}
16116 
16117 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16118   match(If cmp (CmpP (DecodeN oop) zero));
16119   effect(USE labl);
16120 
16121   ins_cost(BRANCH_COST);
16122   format %{ "cb$cmp   $oop, $labl" %}
16123   ins_encode %{
16124     Label* L = $labl$$label;
16125     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16126     if (cond == Assembler::EQ)
16127       __ cbzw($oop$$Register, *L);
16128     else
16129       __ cbnzw($oop$$Register, *L);
16130   %}
16131   ins_pipe(pipe_cmp_branch);
16132 %}
16133 
16134 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16135   match(If cmp (CmpU op1 op2));
16136   effect(USE labl);
16137 
16138   ins_cost(BRANCH_COST);
16139   format %{ "cbw$cmp   $op1, $labl" %}
16140   ins_encode %{
16141     Label* L = $labl$$label;
16142     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16143     if (cond == Assembler::EQ || cond == Assembler::LS)
16144       __ cbzw($op1$$Register, *L);
16145     else
16146       __ cbnzw($op1$$Register, *L);
16147   %}
16148   ins_pipe(pipe_cmp_branch);
16149 %}
16150 
16151 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16152   match(If cmp (CmpUL op1 op2));
16153   effect(USE labl);
16154 
16155   ins_cost(BRANCH_COST);
16156   format %{ "cb$cmp   $op1, $labl" %}
16157   ins_encode %{
16158     Label* L = $labl$$label;
16159     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16160     if (cond == Assembler::EQ || cond == Assembler::LS)
16161       __ cbz($op1$$Register, *L);
16162     else
16163       __ cbnz($op1$$Register, *L);
16164   %}
16165   ins_pipe(pipe_cmp_branch);
16166 %}
16167 
16168 // Test bit and Branch
16169 
16170 // Patterns for short (< 32KiB) variants
16171 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16172   match(If cmp (CmpL op1 op2));
16173   effect(USE labl);
16174 
16175   ins_cost(BRANCH_COST);
16176   format %{ "cb$cmp   $op1, $labl # long" %}
16177   ins_encode %{
16178     Label* L = $labl$$label;
16179     Assembler::Condition cond =
16180       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16181     __ tbr(cond, $op1$$Register, 63, *L);
16182   %}
16183   ins_pipe(pipe_cmp_branch);
16184   ins_short_branch(1);
16185 %}
16186 
16187 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16188   match(If cmp (CmpI op1 op2));
16189   effect(USE labl);
16190 
16191   ins_cost(BRANCH_COST);
16192   format %{ "cb$cmp   $op1, $labl # int" %}
16193   ins_encode %{
16194     Label* L = $labl$$label;
16195     Assembler::Condition cond =
16196       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16197     __ tbr(cond, $op1$$Register, 31, *L);
16198   %}
16199   ins_pipe(pipe_cmp_branch);
16200   ins_short_branch(1);
16201 %}
16202 
16203 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16204   match(If cmp (CmpL (AndL op1 op2) op3));
16205   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16206   effect(USE labl);
16207 
16208   ins_cost(BRANCH_COST);
16209   format %{ "tb$cmp   $op1, $op2, $labl" %}
16210   ins_encode %{
16211     Label* L = $labl$$label;
16212     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16213     int bit = exact_log2_long($op2$$constant);
16214     __ tbr(cond, $op1$$Register, bit, *L);
16215   %}
16216   ins_pipe(pipe_cmp_branch);
16217   ins_short_branch(1);
16218 %}
16219 
16220 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16221   match(If cmp (CmpI (AndI op1 op2) op3));
16222   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16223   effect(USE labl);
16224 
16225   ins_cost(BRANCH_COST);
16226   format %{ "tb$cmp   $op1, $op2, $labl" %}
16227   ins_encode %{
16228     Label* L = $labl$$label;
16229     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16230     int bit = exact_log2((juint)$op2$$constant);
16231     __ tbr(cond, $op1$$Register, bit, *L);
16232   %}
16233   ins_pipe(pipe_cmp_branch);
16234   ins_short_branch(1);
16235 %}
16236 
16237 // And far variants
16238 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16239   match(If cmp (CmpL op1 op2));
16240   effect(USE labl);
16241 
16242   ins_cost(BRANCH_COST);
16243   format %{ "cb$cmp   $op1, $labl # long" %}
16244   ins_encode %{
16245     Label* L = $labl$$label;
16246     Assembler::Condition cond =
16247       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16248     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16249   %}
16250   ins_pipe(pipe_cmp_branch);
16251 %}
16252 
16253 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16254   match(If cmp (CmpI op1 op2));
16255   effect(USE labl);
16256 
16257   ins_cost(BRANCH_COST);
16258   format %{ "cb$cmp   $op1, $labl # int" %}
16259   ins_encode %{
16260     Label* L = $labl$$label;
16261     Assembler::Condition cond =
16262       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16263     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16264   %}
16265   ins_pipe(pipe_cmp_branch);
16266 %}
16267 
16268 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16269   match(If cmp (CmpL (AndL op1 op2) op3));
16270   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16271   effect(USE labl);
16272 
16273   ins_cost(BRANCH_COST);
16274   format %{ "tb$cmp   $op1, $op2, $labl" %}
16275   ins_encode %{
16276     Label* L = $labl$$label;
16277     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16278     int bit = exact_log2_long($op2$$constant);
16279     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16280   %}
16281   ins_pipe(pipe_cmp_branch);
16282 %}
16283 
16284 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16285   match(If cmp (CmpI (AndI op1 op2) op3));
16286   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16287   effect(USE labl);
16288 
16289   ins_cost(BRANCH_COST);
16290   format %{ "tb$cmp   $op1, $op2, $labl" %}
16291   ins_encode %{
16292     Label* L = $labl$$label;
16293     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16294     int bit = exact_log2((juint)$op2$$constant);
16295     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16296   %}
16297   ins_pipe(pipe_cmp_branch);
16298 %}
16299 
16300 // Test bits
16301 
16302 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16303   match(Set cr (CmpL (AndL op1 op2) op3));
16304   predicate(Assembler::operand_valid_for_logical_immediate
16305             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16306 
16307   ins_cost(INSN_COST);
16308   format %{ "tst $op1, $op2 # long" %}
16309   ins_encode %{
16310     __ tst($op1$$Register, $op2$$constant);
16311   %}
16312   ins_pipe(ialu_reg_reg);
16313 %}
16314 
16315 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16316   match(Set cr (CmpI (AndI op1 op2) op3));
16317   predicate(Assembler::operand_valid_for_logical_immediate
16318             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16319 
16320   ins_cost(INSN_COST);
16321   format %{ "tst $op1, $op2 # int" %}
16322   ins_encode %{
16323     __ tstw($op1$$Register, $op2$$constant);
16324   %}
16325   ins_pipe(ialu_reg_reg);
16326 %}
16327 
16328 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16329   match(Set cr (CmpL (AndL op1 op2) op3));
16330 
16331   ins_cost(INSN_COST);
16332   format %{ "tst $op1, $op2 # long" %}
16333   ins_encode %{
16334     __ tst($op1$$Register, $op2$$Register);
16335   %}
16336   ins_pipe(ialu_reg_reg);
16337 %}
16338 
16339 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16340   match(Set cr (CmpI (AndI op1 op2) op3));
16341 
16342   ins_cost(INSN_COST);
16343   format %{ "tstw $op1, $op2 # int" %}
16344   ins_encode %{
16345     __ tstw($op1$$Register, $op2$$Register);
16346   %}
16347   ins_pipe(ialu_reg_reg);
16348 %}
16349 
16350 
16351 // Conditional Far Branch
16352 // Conditional Far Branch Unsigned
16353 // TODO: fixme
16354 
16355 // counted loop end branch near
16356 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16357 %{
16358   match(CountedLoopEnd cmp cr);
16359 
16360   effect(USE lbl);
16361 
16362   ins_cost(BRANCH_COST);
16363   // short variant.
16364   // ins_short_branch(1);
16365   format %{ "b$cmp $lbl \t// counted loop end" %}
16366 
16367   ins_encode(aarch64_enc_br_con(cmp, lbl));
16368 
16369   ins_pipe(pipe_branch);
16370 %}
16371 
16372 // counted loop end branch near Unsigned
16373 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16374 %{
16375   match(CountedLoopEnd cmp cr);
16376 
16377   effect(USE lbl);
16378 
16379   ins_cost(BRANCH_COST);
16380   // short variant.
16381   // ins_short_branch(1);
16382   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16383 
16384   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16385 
16386   ins_pipe(pipe_branch);
16387 %}
16388 
16389 // counted loop end branch far
16390 // counted loop end branch far unsigned
16391 // TODO: fixme
16392 
16393 // ============================================================================
16394 // inlined locking and unlocking
16395 
16396 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16397 %{
16398   match(Set cr (FastLock object box));
16399   effect(TEMP tmp, TEMP tmp2);
16400 
16401   // TODO
16402   // identify correct cost
16403   ins_cost(5 * INSN_COST);
16404   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16405 
16406   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16407 
16408   ins_pipe(pipe_serial);
16409 %}
16410 
16411 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16412 %{
16413   match(Set cr (FastUnlock object box));
16414   effect(TEMP tmp, TEMP tmp2);
16415 
16416   ins_cost(5 * INSN_COST);
16417   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16418 
16419   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16420 
16421   ins_pipe(pipe_serial);
16422 %}
16423 
16424 
16425 // ============================================================================
16426 // Safepoint Instructions
16427 
16428 // TODO
16429 // provide a near and far version of this code
16430 
16431 instruct safePoint(rFlagsReg cr, iRegP poll)
16432 %{
16433   match(SafePoint poll);
16434   effect(KILL cr);
16435 
16436   format %{
16437     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16438   %}
16439   ins_encode %{
16440     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16441   %}
16442   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16443 %}
16444 
16445 
16446 // ============================================================================
16447 // Procedure Call/Return Instructions
16448 
16449 // Call Java Static Instruction
16450 
16451 instruct CallStaticJavaDirect(method meth)
16452 %{
16453   match(CallStaticJava);
16454 
16455   effect(USE meth);
16456 
16457   ins_cost(CALL_COST);
16458 
16459   format %{ "call,static $meth \t// ==> " %}
16460 
16461   ins_encode(aarch64_enc_java_static_call(meth),
16462              aarch64_enc_call_epilog);
16463 
16464   ins_pipe(pipe_class_call);
16465 %}
16466 
16467 // TO HERE
16468 
16469 // Call Java Dynamic Instruction
16470 instruct CallDynamicJavaDirect(method meth)
16471 %{
16472   match(CallDynamicJava);
16473 
16474   effect(USE meth);
16475 
16476   ins_cost(CALL_COST);
16477 
16478   format %{ "CALL,dynamic $meth \t// ==> " %}
16479 
16480   ins_encode(aarch64_enc_java_dynamic_call(meth),
16481              aarch64_enc_call_epilog);
16482 
16483   ins_pipe(pipe_class_call);
16484 %}
16485 
16486 // Call Runtime Instruction
16487 
16488 instruct CallRuntimeDirect(method meth)
16489 %{
16490   match(CallRuntime);
16491 
16492   effect(USE meth);
16493 
16494   ins_cost(CALL_COST);
16495 
16496   format %{ "CALL, runtime $meth" %}
16497 
16498   ins_encode( aarch64_enc_java_to_runtime(meth) );
16499 
16500   ins_pipe(pipe_class_call);
16501 %}
16502 
16503 // Call Runtime Instruction
16504 
16505 instruct CallLeafDirect(method meth)
16506 %{
16507   match(CallLeaf);
16508 
16509   effect(USE meth);
16510 
16511   ins_cost(CALL_COST);
16512 
16513   format %{ "CALL, runtime leaf $meth" %}
16514 
16515   ins_encode( aarch64_enc_java_to_runtime(meth) );
16516 
16517   ins_pipe(pipe_class_call);
16518 %}
16519 
16520 // Call Runtime Instruction
16521 
16522 instruct CallLeafNoFPDirect(method meth)
16523 %{
16524   match(CallLeafNoFP);
16525 
16526   effect(USE meth);
16527 
16528   ins_cost(CALL_COST);
16529 
16530   format %{ "CALL, runtime leaf nofp $meth" %}
16531 
16532   ins_encode( aarch64_enc_java_to_runtime(meth) );
16533 
16534   ins_pipe(pipe_class_call);
16535 %}
16536 
16537 instruct CallNativeDirect(method meth)
16538 %{
16539   match(CallNative);
16540 
16541   effect(USE meth);
16542 
16543   ins_cost(CALL_COST);
16544 
16545   format %{ "CALL, native $meth" %}
16546 
16547   ins_encode( aarch64_enc_java_to_runtime(meth) );
16548 
16549   ins_pipe(pipe_class_call);
16550 %}
16551 
16552 // Tail Call; Jump from runtime stub to Java code.
16553 // Also known as an 'interprocedural jump'.
16554 // Target of jump will eventually return to caller.
16555 // TailJump below removes the return address.
16556 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16557 %{
16558   match(TailCall jump_target method_ptr);
16559 
16560   ins_cost(CALL_COST);
16561 
16562   format %{ "br $jump_target\t# $method_ptr holds method" %}
16563 
16564   ins_encode(aarch64_enc_tail_call(jump_target));
16565 
16566   ins_pipe(pipe_class_call);
16567 %}
16568 
16569 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16570 %{
16571   match(TailJump jump_target ex_oop);
16572 
16573   ins_cost(CALL_COST);
16574 
16575   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16576 
16577   ins_encode(aarch64_enc_tail_jmp(jump_target));
16578 
16579   ins_pipe(pipe_class_call);
16580 %}
16581 
16582 // Create exception oop: created by stack-crawling runtime code.
16583 // Created exception is now available to this handler, and is setup
16584 // just prior to jumping to this handler. No code emitted.
16585 // TODO check
16586 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16587 instruct CreateException(iRegP_R0 ex_oop)
16588 %{
16589   match(Set ex_oop (CreateEx));
16590 
16591   format %{ " -- \t// exception oop; no code emitted" %}
16592 
16593   size(0);
16594 
16595   ins_encode( /*empty*/ );
16596 
16597   ins_pipe(pipe_class_empty);
16598 %}
16599 
16600 // Rethrow exception: The exception oop will come in the first
16601 // argument position. Then JUMP (not call) to the rethrow stub code.
16602 instruct RethrowException() %{
16603   match(Rethrow);
16604   ins_cost(CALL_COST);
16605 
16606   format %{ "b rethrow_stub" %}
16607 
16608   ins_encode( aarch64_enc_rethrow() );
16609 
16610   ins_pipe(pipe_class_call);
16611 %}
16612 
16613 
16614 // Return Instruction
16615 // epilog node loads ret address into lr as part of frame pop
16616 instruct Ret()
16617 %{
16618   match(Return);
16619 
16620   format %{ "ret\t// return register" %}
16621 
16622   ins_encode( aarch64_enc_ret() );
16623 
16624   ins_pipe(pipe_branch);
16625 %}
16626 
16627 // Die now.
16628 instruct ShouldNotReachHere() %{
16629   match(Halt);
16630 
16631   ins_cost(CALL_COST);
16632   format %{ "ShouldNotReachHere" %}
16633 
16634   ins_encode %{
16635     if (is_reachable()) {
16636       __ stop(_halt_reason);
16637     }
16638   %}
16639 
16640   ins_pipe(pipe_class_default);
16641 %}
16642 
16643 // ============================================================================
16644 // Partial Subtype Check
16645 //
16646 // superklass array for an instance of the superklass.  Set a hidden
16647 // internal cache on a hit (cache is checked with exposed code in
16648 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16649 // encoding ALSO sets flags.
16650 
16651 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16652 %{
16653   match(Set result (PartialSubtypeCheck sub super));
16654   effect(KILL cr, KILL temp);
16655 
16656   ins_cost(1100);  // slightly larger than the next version
16657   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16658 
16659   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16660 
16661   opcode(0x1); // Force zero of result reg on hit
16662 
16663   ins_pipe(pipe_class_memory);
16664 %}
16665 
16666 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16667 %{
16668   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16669   effect(KILL temp, KILL result);
16670 
16671   ins_cost(1100);  // slightly larger than the next version
16672   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16673 
16674   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16675 
16676   opcode(0x0); // Don't zero result reg on hit
16677 
16678   ins_pipe(pipe_class_memory);
16679 %}
16680 
16681 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16682                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16683 %{
16684   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16685   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16686   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16687 
16688   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16689   ins_encode %{
16690     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16691     __ string_compare($str1$$Register, $str2$$Register,
16692                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16693                       $tmp1$$Register, $tmp2$$Register,
16694                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16695   %}
16696   ins_pipe(pipe_class_memory);
16697 %}
16698 
16699 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16700                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16701 %{
16702   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16703   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16704   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16705 
16706   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16707   ins_encode %{
16708     __ string_compare($str1$$Register, $str2$$Register,
16709                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16710                       $tmp1$$Register, $tmp2$$Register,
16711                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16712   %}
16713   ins_pipe(pipe_class_memory);
16714 %}
16715 
16716 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16717                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16718                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16719 %{
16720   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16721   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16722   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16723          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16724 
16725   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16726   ins_encode %{
16727     __ string_compare($str1$$Register, $str2$$Register,
16728                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16729                       $tmp1$$Register, $tmp2$$Register,
16730                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16731                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16732   %}
16733   ins_pipe(pipe_class_memory);
16734 %}
16735 
16736 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16737                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16738                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16739 %{
16740   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16741   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16742   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16743          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16744 
16745   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16746   ins_encode %{
16747     __ string_compare($str1$$Register, $str2$$Register,
16748                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16749                       $tmp1$$Register, $tmp2$$Register,
16750                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16751                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16752   %}
16753   ins_pipe(pipe_class_memory);
16754 %}
16755 
16756 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16757        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16758        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16759 %{
16760   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16761   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16762   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16763          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16764   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16765 
16766   ins_encode %{
16767     __ string_indexof($str1$$Register, $str2$$Register,
16768                       $cnt1$$Register, $cnt2$$Register,
16769                       $tmp1$$Register, $tmp2$$Register,
16770                       $tmp3$$Register, $tmp4$$Register,
16771                       $tmp5$$Register, $tmp6$$Register,
16772                       -1, $result$$Register, StrIntrinsicNode::UU);
16773   %}
16774   ins_pipe(pipe_class_memory);
16775 %}
16776 
16777 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16778        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16779        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16780 %{
16781   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16782   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16783   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16784          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16785   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16786 
16787   ins_encode %{
16788     __ string_indexof($str1$$Register, $str2$$Register,
16789                       $cnt1$$Register, $cnt2$$Register,
16790                       $tmp1$$Register, $tmp2$$Register,
16791                       $tmp3$$Register, $tmp4$$Register,
16792                       $tmp5$$Register, $tmp6$$Register,
16793                       -1, $result$$Register, StrIntrinsicNode::LL);
16794   %}
16795   ins_pipe(pipe_class_memory);
16796 %}
16797 
16798 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16799        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16800        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16801 %{
16802   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16803   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16804   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16805          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16806   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16807 
16808   ins_encode %{
16809     __ string_indexof($str1$$Register, $str2$$Register,
16810                       $cnt1$$Register, $cnt2$$Register,
16811                       $tmp1$$Register, $tmp2$$Register,
16812                       $tmp3$$Register, $tmp4$$Register,
16813                       $tmp5$$Register, $tmp6$$Register,
16814                       -1, $result$$Register, StrIntrinsicNode::UL);
16815   %}
16816   ins_pipe(pipe_class_memory);
16817 %}
16818 
16819 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16820                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16821                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16822 %{
16823   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16824   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16825   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16826          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16827   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16828 
16829   ins_encode %{
16830     int icnt2 = (int)$int_cnt2$$constant;
16831     __ string_indexof($str1$$Register, $str2$$Register,
16832                       $cnt1$$Register, zr,
16833                       $tmp1$$Register, $tmp2$$Register,
16834                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16835                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16836   %}
16837   ins_pipe(pipe_class_memory);
16838 %}
16839 
16840 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16841                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16842                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16843 %{
16844   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16845   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16846   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16847          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16848   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16849 
16850   ins_encode %{
16851     int icnt2 = (int)$int_cnt2$$constant;
16852     __ string_indexof($str1$$Register, $str2$$Register,
16853                       $cnt1$$Register, zr,
16854                       $tmp1$$Register, $tmp2$$Register,
16855                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16856                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16857   %}
16858   ins_pipe(pipe_class_memory);
16859 %}
16860 
16861 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16862                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16863                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16864 %{
16865   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16866   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16867   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16868          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16869   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16870 
16871   ins_encode %{
16872     int icnt2 = (int)$int_cnt2$$constant;
16873     __ string_indexof($str1$$Register, $str2$$Register,
16874                       $cnt1$$Register, zr,
16875                       $tmp1$$Register, $tmp2$$Register,
16876                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16877                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16878   %}
16879   ins_pipe(pipe_class_memory);
16880 %}
16881 
16882 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16883                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16884                              iRegINoSp tmp3, rFlagsReg cr)
16885 %{
16886   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16887   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16888   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16889          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16890 
16891   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16892 
16893   ins_encode %{
16894     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16895                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16896                            $tmp3$$Register);
16897   %}
16898   ins_pipe(pipe_class_memory);
16899 %}
16900 
16901 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16902                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16903                               iRegINoSp tmp3, rFlagsReg cr)
16904 %{
16905   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16906   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16907   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16908          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16909 
16910   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16911 
16912   ins_encode %{
16913     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16914                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16915                             $tmp3$$Register);
16916   %}
16917   ins_pipe(pipe_class_memory);
16918 %}
16919 
16920 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16921                         iRegI_R0 result, rFlagsReg cr)
16922 %{
16923   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16924   match(Set result (StrEquals (Binary str1 str2) cnt));
16925   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16926 
16927   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16928   ins_encode %{
16929     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16930     __ string_equals($str1$$Register, $str2$$Register,
16931                      $result$$Register, $cnt$$Register, 1);
16932   %}
16933   ins_pipe(pipe_class_memory);
16934 %}
16935 
16936 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16937                         iRegI_R0 result, rFlagsReg cr)
16938 %{
16939   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16940   match(Set result (StrEquals (Binary str1 str2) cnt));
16941   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16942 
16943   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16944   ins_encode %{
16945     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16946     __ string_equals($str1$$Register, $str2$$Register,
16947                      $result$$Register, $cnt$$Register, 2);
16948   %}
16949   ins_pipe(pipe_class_memory);
16950 %}
16951 
16952 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16953                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16954                        iRegP_R10 tmp, rFlagsReg cr)
16955 %{
16956   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16957   match(Set result (AryEq ary1 ary2));
16958   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16959 
16960   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16961   ins_encode %{
16962     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16963                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16964                                    $result$$Register, $tmp$$Register, 1);
16965     if (tpc == NULL) {
16966       ciEnv::current()->record_failure("CodeCache is full");
16967       return;
16968     }
16969   %}
16970   ins_pipe(pipe_class_memory);
16971 %}
16972 
16973 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16974                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16975                        iRegP_R10 tmp, rFlagsReg cr)
16976 %{
16977   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16978   match(Set result (AryEq ary1 ary2));
16979   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16980 
16981   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16982   ins_encode %{
16983     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16984                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16985                                    $result$$Register, $tmp$$Register, 2);
16986     if (tpc == NULL) {
16987       ciEnv::current()->record_failure("CodeCache is full");
16988       return;
16989     }
16990   %}
16991   ins_pipe(pipe_class_memory);
16992 %}
16993 
16994 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16995 %{
16996   match(Set result (HasNegatives ary1 len));
16997   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16998   format %{ "has negatives byte[] $ary1,$len -> $result" %}
16999   ins_encode %{
17000     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
17001     if (tpc == NULL) {
17002       ciEnv::current()->record_failure("CodeCache is full");
17003       return;
17004     }
17005   %}
17006   ins_pipe( pipe_slow );
17007 %}
17008 
17009 // fast char[] to byte[] compression
17010 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17011                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17012                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17013                          iRegI_R0 result, rFlagsReg cr)
17014 %{
17015   match(Set result (StrCompressedCopy src (Binary dst len)));
17016   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17017 
17018   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
17019   ins_encode %{
17020     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17021                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17022                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
17023                            $result$$Register);
17024   %}
17025   ins_pipe( pipe_slow );
17026 %}
17027 
17028 // fast byte[] to char[] inflation
17029 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17030                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17031 %{
17032   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17033   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17034 
17035   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17036   ins_encode %{
17037     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17038                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17039                                         $tmp3$$FloatRegister, $tmp4$$Register);
17040     if (tpc == NULL) {
17041       ciEnv::current()->record_failure("CodeCache is full");
17042       return;
17043     }
17044   %}
17045   ins_pipe(pipe_class_memory);
17046 %}
17047 
17048 // encode char[] to byte[] in ISO_8859_1
17049 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17050                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
17051                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
17052                           iRegI_R0 result, rFlagsReg cr)
17053 %{
17054   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17055   match(Set result (EncodeISOArray src (Binary dst len)));
17056   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
17057          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
17058 
17059   format %{ "Encode array $src,$dst,$len -> $result" %}
17060   ins_encode %{
17061     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17062          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
17063          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
17064   %}
17065   ins_pipe( pipe_class_memory );
17066 %}
17067 
17068 // ============================================================================
17069 // This name is KNOWN by the ADLC and cannot be changed.
17070 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17071 // for this guy.
17072 instruct tlsLoadP(thread_RegP dst)
17073 %{
17074   match(Set dst (ThreadLocal));
17075 
17076   ins_cost(0);
17077 
17078   format %{ " -- \t// $dst=Thread::current(), empty" %}
17079 
17080   size(0);
17081 
17082   ins_encode( /*empty*/ );
17083 
17084   ins_pipe(pipe_class_empty);
17085 %}
17086 
17087 //----------PEEPHOLE RULES-----------------------------------------------------
17088 // These must follow all instruction definitions as they use the names
17089 // defined in the instructions definitions.
17090 //
17091 // peepmatch ( root_instr_name [preceding_instruction]* );
17092 //
17093 // peepconstraint %{
17094 // (instruction_number.operand_name relational_op instruction_number.operand_name
17095 //  [, ...] );
17096 // // instruction numbers are zero-based using left to right order in peepmatch
17097 //
17098 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17099 // // provide an instruction_number.operand_name for each operand that appears
17100 // // in the replacement instruction's match rule
17101 //
17102 // ---------VM FLAGS---------------------------------------------------------
17103 //
17104 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17105 //
17106 // Each peephole rule is given an identifying number starting with zero and
17107 // increasing by one in the order seen by the parser.  An individual peephole
17108 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17109 // on the command-line.
17110 //
17111 // ---------CURRENT LIMITATIONS----------------------------------------------
17112 //
17113 // Only match adjacent instructions in same basic block
17114 // Only equality constraints
17115 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17116 // Only one replacement instruction
17117 //
17118 // ---------EXAMPLE----------------------------------------------------------
17119 //
17120 // // pertinent parts of existing instructions in architecture description
17121 // instruct movI(iRegINoSp dst, iRegI src)
17122 // %{
17123 //   match(Set dst (CopyI src));
17124 // %}
17125 //
17126 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17127 // %{
17128 //   match(Set dst (AddI dst src));
17129 //   effect(KILL cr);
17130 // %}
17131 //
17132 // // Change (inc mov) to lea
17133 // peephole %{
17134 //   // increment preceeded by register-register move
17135 //   peepmatch ( incI_iReg movI );
17136 //   // require that the destination register of the increment
17137 //   // match the destination register of the move
17138 //   peepconstraint ( 0.dst == 1.dst );
17139 //   // construct a replacement instruction that sets
17140 //   // the destination to ( move's source register + one )
17141 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17142 // %}
17143 //
17144 
17145 // Implementation no longer uses movX instructions since
17146 // machine-independent system no longer uses CopyX nodes.
17147 //
17148 // peephole
17149 // %{
17150 //   peepmatch (incI_iReg movI);
17151 //   peepconstraint (0.dst == 1.dst);
17152 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17153 // %}
17154 
17155 // peephole
17156 // %{
17157 //   peepmatch (decI_iReg movI);
17158 //   peepconstraint (0.dst == 1.dst);
17159 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17160 // %}
17161 
17162 // peephole
17163 // %{
17164 //   peepmatch (addI_iReg_imm movI);
17165 //   peepconstraint (0.dst == 1.dst);
17166 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17167 // %}
17168 
17169 // peephole
17170 // %{
17171 //   peepmatch (incL_iReg movL);
17172 //   peepconstraint (0.dst == 1.dst);
17173 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17174 // %}
17175 
17176 // peephole
17177 // %{
17178 //   peepmatch (decL_iReg movL);
17179 //   peepconstraint (0.dst == 1.dst);
17180 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17181 // %}
17182 
17183 // peephole
17184 // %{
17185 //   peepmatch (addL_iReg_imm movL);
17186 //   peepconstraint (0.dst == 1.dst);
17187 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17188 // %}
17189 
17190 // peephole
17191 // %{
17192 //   peepmatch (addP_iReg_imm movP);
17193 //   peepconstraint (0.dst == 1.dst);
17194 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17195 // %}
17196 
17197 // // Change load of spilled value to only a spill
17198 // instruct storeI(memory mem, iRegI src)
17199 // %{
17200 //   match(Set mem (StoreI mem src));
17201 // %}
17202 //
17203 // instruct loadI(iRegINoSp dst, memory mem)
17204 // %{
17205 //   match(Set dst (LoadI mem));
17206 // %}
17207 //
17208 
17209 //----------SMARTSPILL RULES---------------------------------------------------
17210 // These must follow all instruction definitions as they use the names
17211 // defined in the instructions definitions.
17212 
17213 // Local Variables:
17214 // mode: c++
17215 // End:
--- EOF ---