1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_codestub_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298  bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // predicate controlling addressing modes
 1315   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1316 %}
 1317 
 1318 source %{
 1319 
 1320   // Derived RegMask with conditionally allocatable registers
 1321 
 1322   void PhaseOutput::pd_perform_mach_node_analysis() {
 1323   }
 1324 
 1325   int MachNode::pd_alignment_required() const {
 1326     return 1;
 1327   }
 1328 
 1329   int MachNode::compute_padding(int current_offset) const {
 1330     return 0;
 1331   }
 1332 
 1333   RegMask _ANY_REG32_mask;
 1334   RegMask _ANY_REG_mask;
 1335   RegMask _PTR_REG_mask;
 1336   RegMask _NO_SPECIAL_REG32_mask;
 1337   RegMask _NO_SPECIAL_REG_mask;
 1338   RegMask _NO_SPECIAL_PTR_REG_mask;
 1339 
 1340   void reg_mask_init() {
 1341     // We derive below RegMask(s) from the ones which are auto-generated from
 1342     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1343     // registers conditionally reserved.
 1344 
 1345     _ANY_REG32_mask = _ALL_REG32_mask;
 1346     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1347 
 1348     _ANY_REG_mask = _ALL_REG_mask;
 1349 
 1350     _PTR_REG_mask = _ALL_REG_mask;
 1351 
 1352     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1353     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1354 
 1355     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1356     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1357 
 1358     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     // r27 is not allocatable when compressed oops is on and heapbase is not
 1362     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1363     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1364       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1365       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1366       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1367     }
 1368 
 1369     // r29 is not allocatable when PreserveFramePointer is on
 1370     if (PreserveFramePointer) {
 1371       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1372       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1373       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1374     }
 1375   }
 1376 
 1377   // Optimizaton of volatile gets and puts
 1378   // -------------------------------------
 1379   //
 1380   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1381   // use to implement volatile reads and writes. For a volatile read
 1382   // we simply need
 1383   //
 1384   //   ldar<x>
 1385   //
 1386   // and for a volatile write we need
 1387   //
 1388   //   stlr<x>
 1389   //
 1390   // Alternatively, we can implement them by pairing a normal
 1391   // load/store with a memory barrier. For a volatile read we need
 1392   //
 1393   //   ldr<x>
 1394   //   dmb ishld
 1395   //
 1396   // for a volatile write
 1397   //
 1398   //   dmb ish
 1399   //   str<x>
 1400   //   dmb ish
 1401   //
 1402   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1403   // sequences. These are normally translated to an instruction
 1404   // sequence like the following
 1405   //
 1406   //   dmb      ish
 1407   // retry:
 1408   //   ldxr<x>   rval raddr
 1409   //   cmp       rval rold
 1410   //   b.ne done
 1411   //   stlxr<x>  rval, rnew, rold
 1412   //   cbnz      rval retry
 1413   // done:
 1414   //   cset      r0, eq
 1415   //   dmb ishld
 1416   //
 1417   // Note that the exclusive store is already using an stlxr
 1418   // instruction. That is required to ensure visibility to other
 1419   // threads of the exclusive write (assuming it succeeds) before that
 1420   // of any subsequent writes.
 1421   //
 1422   // The following instruction sequence is an improvement on the above
 1423   //
 1424   // retry:
 1425   //   ldaxr<x>  rval raddr
 1426   //   cmp       rval rold
 1427   //   b.ne done
 1428   //   stlxr<x>  rval, rnew, rold
 1429   //   cbnz      rval retry
 1430   // done:
 1431   //   cset      r0, eq
 1432   //
 1433   // We don't need the leading dmb ish since the stlxr guarantees
 1434   // visibility of prior writes in the case that the swap is
 1435   // successful. Crucially we don't have to worry about the case where
 1436   // the swap is not successful since no valid program should be
 1437   // relying on visibility of prior changes by the attempting thread
 1438   // in the case where the CAS fails.
 1439   //
 1440   // Similarly, we don't need the trailing dmb ishld if we substitute
 1441   // an ldaxr instruction since that will provide all the guarantees we
 1442   // require regarding observation of changes made by other threads
 1443   // before any change to the CAS address observed by the load.
 1444   //
 1445   // In order to generate the desired instruction sequence we need to
 1446   // be able to identify specific 'signature' ideal graph node
 1447   // sequences which i) occur as a translation of a volatile reads or
 1448   // writes or CAS operations and ii) do not occur through any other
 1449   // translation or graph transformation. We can then provide
 1450   // alternative aldc matching rules which translate these node
 1451   // sequences to the desired machine code sequences. Selection of the
 1452   // alternative rules can be implemented by predicates which identify
 1453   // the relevant node sequences.
 1454   //
 1455   // The ideal graph generator translates a volatile read to the node
 1456   // sequence
 1457   //
 1458   //   LoadX[mo_acquire]
 1459   //   MemBarAcquire
 1460   //
 1461   // As a special case when using the compressed oops optimization we
 1462   // may also see this variant
 1463   //
 1464   //   LoadN[mo_acquire]
 1465   //   DecodeN
 1466   //   MemBarAcquire
 1467   //
 1468   // A volatile write is translated to the node sequence
 1469   //
 1470   //   MemBarRelease
 1471   //   StoreX[mo_release] {CardMark}-optional
 1472   //   MemBarVolatile
 1473   //
 1474   // n.b. the above node patterns are generated with a strict
 1475   // 'signature' configuration of input and output dependencies (see
 1476   // the predicates below for exact details). The card mark may be as
 1477   // simple as a few extra nodes or, in a few GC configurations, may
 1478   // include more complex control flow between the leading and
 1479   // trailing memory barriers. However, whatever the card mark
 1480   // configuration these signatures are unique to translated volatile
 1481   // reads/stores -- they will not appear as a result of any other
 1482   // bytecode translation or inlining nor as a consequence of
 1483   // optimizing transforms.
 1484   //
 1485   // We also want to catch inlined unsafe volatile gets and puts and
 1486   // be able to implement them using either ldar<x>/stlr<x> or some
 1487   // combination of ldr<x>/stlr<x> and dmb instructions.
 1488   //
 1489   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1490   // normal volatile put node sequence containing an extra cpuorder
 1491   // membar
 1492   //
 1493   //   MemBarRelease
 1494   //   MemBarCPUOrder
 1495   //   StoreX[mo_release] {CardMark}-optional
 1496   //   MemBarCPUOrder
 1497   //   MemBarVolatile
 1498   //
 1499   // n.b. as an aside, a cpuorder membar is not itself subject to
 1500   // matching and translation by adlc rules.  However, the rule
 1501   // predicates need to detect its presence in order to correctly
 1502   // select the desired adlc rules.
 1503   //
 1504   // Inlined unsafe volatile gets manifest as a slightly different
 1505   // node sequence to a normal volatile get because of the
 1506   // introduction of some CPUOrder memory barriers to bracket the
 1507   // Load. However, but the same basic skeleton of a LoadX feeding a
 1508   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1509   // present
 1510   //
 1511   //   MemBarCPUOrder
 1512   //        ||       \\
 1513   //   MemBarCPUOrder LoadX[mo_acquire]
 1514   //        ||            |
 1515   //        ||       {DecodeN} optional
 1516   //        ||       /
 1517   //     MemBarAcquire
 1518   //
 1519   // In this case the acquire membar does not directly depend on the
 1520   // load. However, we can be sure that the load is generated from an
 1521   // inlined unsafe volatile get if we see it dependent on this unique
 1522   // sequence of membar nodes. Similarly, given an acquire membar we
 1523   // can know that it was added because of an inlined unsafe volatile
 1524   // get if it is fed and feeds a cpuorder membar and if its feed
 1525   // membar also feeds an acquiring load.
 1526   //
 1527   // Finally an inlined (Unsafe) CAS operation is translated to the
 1528   // following ideal graph
 1529   //
 1530   //   MemBarRelease
 1531   //   MemBarCPUOrder
 1532   //   CompareAndSwapX {CardMark}-optional
 1533   //   MemBarCPUOrder
 1534   //   MemBarAcquire
 1535   //
 1536   // So, where we can identify these volatile read and write
 1537   // signatures we can choose to plant either of the above two code
 1538   // sequences. For a volatile read we can simply plant a normal
 1539   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1540   // also choose to inhibit translation of the MemBarAcquire and
 1541   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1542   //
 1543   // When we recognise a volatile store signature we can choose to
 1544   // plant at a dmb ish as a translation for the MemBarRelease, a
 1545   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1546   // Alternatively, we can inhibit translation of the MemBarRelease
 1547   // and MemBarVolatile and instead plant a simple stlr<x>
 1548   // instruction.
 1549   //
 1550   // when we recognise a CAS signature we can choose to plant a dmb
 1551   // ish as a translation for the MemBarRelease, the conventional
 1552   // macro-instruction sequence for the CompareAndSwap node (which
 1553   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1554   // Alternatively, we can elide generation of the dmb instructions
 1555   // and plant the alternative CompareAndSwap macro-instruction
 1556   // sequence (which uses ldaxr<x>).
 1557   //
 1558   // Of course, the above only applies when we see these signature
 1559   // configurations. We still want to plant dmb instructions in any
 1560   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1561   // MemBarVolatile. For example, at the end of a constructor which
 1562   // writes final/volatile fields we will see a MemBarRelease
 1563   // instruction and this needs a 'dmb ish' lest we risk the
 1564   // constructed object being visible without making the
 1565   // final/volatile field writes visible.
 1566   //
 1567   // n.b. the translation rules below which rely on detection of the
 1568   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1569   // If we see anything other than the signature configurations we
 1570   // always just translate the loads and stores to ldr<x> and str<x>
 1571   // and translate acquire, release and volatile membars to the
 1572   // relevant dmb instructions.
 1573   //
 1574 
 1575   // is_CAS(int opcode, bool maybe_volatile)
 1576   //
 1577   // return true if opcode is one of the possible CompareAndSwapX
 1578   // values otherwise false.
 1579 
 1580   bool is_CAS(int opcode, bool maybe_volatile)
 1581   {
 1582     switch(opcode) {
 1583       // We handle these
 1584     case Op_CompareAndSwapI:
 1585     case Op_CompareAndSwapL:
 1586     case Op_CompareAndSwapP:
 1587     case Op_CompareAndSwapN:
 1588     case Op_ShenandoahCompareAndSwapP:
 1589     case Op_ShenandoahCompareAndSwapN:
 1590     case Op_CompareAndSwapB:
 1591     case Op_CompareAndSwapS:
 1592     case Op_GetAndSetI:
 1593     case Op_GetAndSetL:
 1594     case Op_GetAndSetP:
 1595     case Op_GetAndSetN:
 1596     case Op_GetAndAddI:
 1597     case Op_GetAndAddL:
 1598       return true;
 1599     case Op_CompareAndExchangeI:
 1600     case Op_CompareAndExchangeN:
 1601     case Op_CompareAndExchangeB:
 1602     case Op_CompareAndExchangeS:
 1603     case Op_CompareAndExchangeL:
 1604     case Op_CompareAndExchangeP:
 1605     case Op_WeakCompareAndSwapB:
 1606     case Op_WeakCompareAndSwapS:
 1607     case Op_WeakCompareAndSwapI:
 1608     case Op_WeakCompareAndSwapL:
 1609     case Op_WeakCompareAndSwapP:
 1610     case Op_WeakCompareAndSwapN:
 1611     case Op_ShenandoahWeakCompareAndSwapP:
 1612     case Op_ShenandoahWeakCompareAndSwapN:
 1613     case Op_ShenandoahCompareAndExchangeP:
 1614     case Op_ShenandoahCompareAndExchangeN:
 1615       return maybe_volatile;
 1616     default:
 1617       return false;
 1618     }
 1619   }
 1620 
 1621   // helper to determine the maximum number of Phi nodes we may need to
 1622   // traverse when searching from a card mark membar for the merge mem
 1623   // feeding a trailing membar or vice versa
 1624 
 1625 // predicates controlling emit of ldr<x>/ldar<x>
 1626 
 1627 bool unnecessary_acquire(const Node *barrier)
 1628 {
 1629   assert(barrier->is_MemBar(), "expecting a membar");
 1630 
 1631   MemBarNode* mb = barrier->as_MemBar();
 1632 
 1633   if (mb->trailing_load()) {
 1634     return true;
 1635   }
 1636 
 1637   if (mb->trailing_load_store()) {
 1638     Node* load_store = mb->in(MemBarNode::Precedent);
 1639     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1640     return is_CAS(load_store->Opcode(), true);
 1641   }
 1642 
 1643   return false;
 1644 }
 1645 
 1646 bool needs_acquiring_load(const Node *n)
 1647 {
 1648   assert(n->is_Load(), "expecting a load");
 1649   LoadNode *ld = n->as_Load();
 1650   return ld->is_acquire();
 1651 }
 1652 
 1653 bool unnecessary_release(const Node *n)
 1654 {
 1655   assert((n->is_MemBar() &&
 1656           n->Opcode() == Op_MemBarRelease),
 1657          "expecting a release membar");
 1658 
 1659   MemBarNode *barrier = n->as_MemBar();
 1660   if (!barrier->leading()) {
 1661     return false;
 1662   } else {
 1663     Node* trailing = barrier->trailing_membar();
 1664     MemBarNode* trailing_mb = trailing->as_MemBar();
 1665     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1666     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1667 
 1668     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1669     if (mem->is_Store()) {
 1670       assert(mem->as_Store()->is_release(), "");
 1671       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1672       return true;
 1673     } else {
 1674       assert(mem->is_LoadStore(), "");
 1675       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1676       return is_CAS(mem->Opcode(), true);
 1677     }
 1678   }
 1679   return false;
 1680 }
 1681 
 1682 bool unnecessary_volatile(const Node *n)
 1683 {
 1684   // assert n->is_MemBar();
 1685   MemBarNode *mbvol = n->as_MemBar();
 1686 
 1687   bool release = mbvol->trailing_store();
 1688   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1689 #ifdef ASSERT
 1690   if (release) {
 1691     Node* leading = mbvol->leading_membar();
 1692     assert(leading->Opcode() == Op_MemBarRelease, "");
 1693     assert(leading->as_MemBar()->leading_store(), "");
 1694     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1695   }
 1696 #endif
 1697 
 1698   return release;
 1699 }
 1700 
 1701 // predicates controlling emit of str<x>/stlr<x>
 1702 
 1703 bool needs_releasing_store(const Node *n)
 1704 {
 1705   // assert n->is_Store();
 1706   StoreNode *st = n->as_Store();
 1707   return st->trailing_membar() != NULL;
 1708 }
 1709 
 1710 // predicate controlling translation of CAS
 1711 //
 1712 // returns true if CAS needs to use an acquiring load otherwise false
 1713 
 1714 bool needs_acquiring_load_exclusive(const Node *n)
 1715 {
 1716   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1717   LoadStoreNode* ldst = n->as_LoadStore();
 1718   if (is_CAS(n->Opcode(), false)) {
 1719     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1720   } else {
 1721     return ldst->trailing_membar() != NULL;
 1722   }
 1723 
 1724   // so we can just return true here
 1725   return true;
 1726 }
 1727 
 1728 #define __ _masm.
 1729 
 1730 // advance declarations for helper functions to convert register
 1731 // indices to register objects
 1732 
 1733 // the ad file has to provide implementations of certain methods
 1734 // expected by the generic code
 1735 //
 1736 // REQUIRED FUNCTIONALITY
 1737 
 1738 //=============================================================================
 1739 
 1740 // !!!!! Special hack to get all types of calls to specify the byte offset
 1741 //       from the start of the call to the point where the return address
 1742 //       will point.
 1743 
 1744 int MachCallStaticJavaNode::ret_addr_offset()
 1745 {
 1746   // call should be a simple bl
 1747   int off = 4;
 1748   return off;
 1749 }
 1750 
 1751 int MachCallDynamicJavaNode::ret_addr_offset()
 1752 {
 1753   return 16; // movz, movk, movk, bl
 1754 }
 1755 
 1756 int MachCallRuntimeNode::ret_addr_offset() {
 1757   // for generated stubs the call will be
 1758   //   bl(addr)
 1759   // or with far branches
 1760   //   bl(trampoline_stub)
 1761   // for real runtime callouts it will be six instructions
 1762   // see aarch64_enc_java_to_runtime
 1763   //   adr(rscratch2, retaddr)
 1764   //   lea(rscratch1, RuntimeAddress(addr)
 1765   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1766   //   blr(rscratch1)
 1767   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1768   if (cb) {
 1769     return 1 * NativeInstruction::instruction_size;
 1770   } else {
 1771     return 6 * NativeInstruction::instruction_size;
 1772   }
 1773 }
 1774 
 1775 int MachCallNativeNode::ret_addr_offset() {
 1776   // This is implemented using aarch64_enc_java_to_runtime as above.
 1777   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1778   if (cb) {
 1779     return 1 * NativeInstruction::instruction_size;
 1780   } else {
 1781     return 6 * NativeInstruction::instruction_size;
 1782   }
 1783 }
 1784 
 1785 //=============================================================================
 1786 
 1787 #ifndef PRODUCT
 1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1789   st->print("BREAKPOINT");
 1790 }
 1791 #endif
 1792 
 1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1794   C2_MacroAssembler _masm(&cbuf);
 1795   __ brk(0);
 1796 }
 1797 
 1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1799   return MachNode::size(ra_);
 1800 }
 1801 
 1802 //=============================================================================
 1803 
 1804 #ifndef PRODUCT
 1805   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1806     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1807   }
 1808 #endif
 1809 
 1810   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1811     C2_MacroAssembler _masm(&cbuf);
 1812     for (int i = 0; i < _count; i++) {
 1813       __ nop();
 1814     }
 1815   }
 1816 
 1817   uint MachNopNode::size(PhaseRegAlloc*) const {
 1818     return _count * NativeInstruction::instruction_size;
 1819   }
 1820 
 1821 //=============================================================================
 1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1823 
 1824 int ConstantTable::calculate_table_base_offset() const {
 1825   return 0;  // absolute addressing, no offset
 1826 }
 1827 
 1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1830   ShouldNotReachHere();
 1831 }
 1832 
 1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1834   // Empty encoding
 1835 }
 1836 
 1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1838   return 0;
 1839 }
 1840 
 1841 #ifndef PRODUCT
 1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1843   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1844 }
 1845 #endif
 1846 
 1847 #ifndef PRODUCT
 1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1849   Compile* C = ra_->C;
 1850 
 1851   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1852 
 1853   if (C->output()->need_stack_bang(framesize))
 1854     st->print("# stack bang size=%d\n\t", framesize);
 1855 
 1856   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1857     st->print("sub  sp, sp, #%d\n\t", framesize);
 1858     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1859     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1860   } else {
 1861     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1862     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1863     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1864     st->print("sub  sp, sp, rscratch1");
 1865   }
 1866   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1867     st->print("\n\t");
 1868     st->print("ldr  rscratch1, [guard]\n\t");
 1869     st->print("dmb ishld\n\t");
 1870     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1871     st->print("cmp  rscratch1, rscratch2\n\t");
 1872     st->print("b.eq skip");
 1873     st->print("\n\t");
 1874     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1875     st->print("b skip\n\t");
 1876     st->print("guard: int\n\t");
 1877     st->print("\n\t");
 1878     st->print("skip:\n\t");
 1879   }
 1880 }
 1881 #endif
 1882 
 1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1884   Compile* C = ra_->C;
 1885   C2_MacroAssembler _masm(&cbuf);
 1886 
 1887   // n.b. frame size includes space for return pc and rfp
 1888   const int framesize = C->output()->frame_size_in_bytes();
 1889 
 1890   // insert a nop at the start of the prolog so we can patch in a
 1891   // branch if we need to invalidate the method later
 1892   __ nop();
 1893 
 1894   if (C->clinit_barrier_on_entry()) {
 1895     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1896 
 1897     Label L_skip_barrier;
 1898 
 1899     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1900     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1901     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1902     __ bind(L_skip_barrier);
 1903   }
 1904 
 1905   if (C->max_vector_size() >= 16) {
 1906     __ reinitialize_ptrue();
 1907   }
 1908 
 1909   int bangsize = C->output()->bang_size_in_bytes();
 1910   if (C->output()->need_stack_bang(bangsize))
 1911     __ generate_stack_overflow_check(bangsize);
 1912 
 1913   __ build_frame(framesize);
 1914 
 1915   if (C->stub_function() == NULL) {
 1916     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1917     bs->nmethod_entry_barrier(&_masm);
 1918   }
 1919 
 1920   if (VerifyStackAtCalls) {
 1921     Unimplemented();
 1922   }
 1923 
 1924   C->output()->set_frame_complete(cbuf.insts_size());
 1925 
 1926   if (C->has_mach_constant_base_node()) {
 1927     // NOTE: We set the table base offset here because users might be
 1928     // emitted before MachConstantBaseNode.
 1929     ConstantTable& constant_table = C->output()->constant_table();
 1930     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1931   }
 1932 }
 1933 
 1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1935 {
 1936   return MachNode::size(ra_); // too many variables; just compute it
 1937                               // the hard way
 1938 }
 1939 
 1940 int MachPrologNode::reloc() const
 1941 {
 1942   return 0;
 1943 }
 1944 
 1945 //=============================================================================
 1946 
 1947 #ifndef PRODUCT
 1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1949   Compile* C = ra_->C;
 1950   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1951 
 1952   st->print("# pop frame %d\n\t",framesize);
 1953 
 1954   if (framesize == 0) {
 1955     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1956   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1957     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1958     st->print("add  sp, sp, #%d\n\t", framesize);
 1959   } else {
 1960     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, rscratch1\n\t");
 1962     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1963   }
 1964 
 1965   if (do_polling() && C->is_method_compilation()) {
 1966     st->print("# test polling word\n\t");
 1967     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1968     st->print("cmp  sp, rscratch1\n\t");
 1969     st->print("bhi #slow_path");
 1970   }
 1971 }
 1972 #endif
 1973 
 1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1975   Compile* C = ra_->C;
 1976   C2_MacroAssembler _masm(&cbuf);
 1977   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1978 
 1979   __ remove_frame(framesize);
 1980 
 1981   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1982     __ reserved_stack_check();
 1983   }
 1984 
 1985   if (do_polling() && C->is_method_compilation()) {
 1986     Label dummy_label;
 1987     Label* code_stub = &dummy_label;
 1988     if (!C->output()->in_scratch_emit_size()) {
 1989       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1990       C->output()->add_stub(stub);
 1991       code_stub = &stub->entry();
 1992     }
 1993     __ relocate(relocInfo::poll_return_type);
 1994     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1995   }
 1996 }
 1997 
 1998 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1999   // Variable size. Determine dynamically.
 2000   return MachNode::size(ra_);
 2001 }
 2002 
 2003 int MachEpilogNode::reloc() const {
 2004   // Return number of relocatable values contained in this instruction.
 2005   return 1; // 1 for polling page.
 2006 }
 2007 
 2008 const Pipeline * MachEpilogNode::pipeline() const {
 2009   return MachNode::pipeline_class();
 2010 }
 2011 
 2012 //=============================================================================
 2013 
 2014 // Figure out which register class each belongs in: rc_int, rc_float or
 2015 // rc_stack.
 2016 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2017 
 2018 static enum RC rc_class(OptoReg::Name reg) {
 2019 
 2020   if (reg == OptoReg::Bad) {
 2021     return rc_bad;
 2022   }
 2023 
 2024   // we have 32 int registers * 2 halves
 2025   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2026 
 2027   if (reg < slots_of_int_registers) {
 2028     return rc_int;
 2029   }
 2030 
 2031   // we have 32 float register * 8 halves
 2032   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2033   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2034     return rc_float;
 2035   }
 2036 
 2037   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2038   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2039     return rc_predicate;
 2040   }
 2041 
 2042   // Between predicate regs & stack is the flags.
 2043   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2044 
 2045   return rc_stack;
 2046 }
 2047 
 2048 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2049   Compile* C = ra_->C;
 2050 
 2051   // Get registers to move.
 2052   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2053   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2054   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2055   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2056 
 2057   enum RC src_hi_rc = rc_class(src_hi);
 2058   enum RC src_lo_rc = rc_class(src_lo);
 2059   enum RC dst_hi_rc = rc_class(dst_hi);
 2060   enum RC dst_lo_rc = rc_class(dst_lo);
 2061 
 2062   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2063 
 2064   if (src_hi != OptoReg::Bad) {
 2065     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2066            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2067            "expected aligned-adjacent pairs");
 2068   }
 2069 
 2070   if (src_lo == dst_lo && src_hi == dst_hi) {
 2071     return 0;            // Self copy, no move.
 2072   }
 2073 
 2074   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2075               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2076   int src_offset = ra_->reg2offset(src_lo);
 2077   int dst_offset = ra_->reg2offset(dst_lo);
 2078 
 2079   if (bottom_type()->isa_vect() != NULL) {
 2080     uint ireg = ideal_reg();
 2081     if (ireg == Op_VecA && cbuf) {
 2082       C2_MacroAssembler _masm(cbuf);
 2083       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2084       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2085         // stack->stack
 2086         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2087                                                 sve_vector_reg_size_in_bytes);
 2088       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2089         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2090                             sve_vector_reg_size_in_bytes);
 2091       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2092         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2093                               sve_vector_reg_size_in_bytes);
 2094       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2095         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2096                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2097                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2098       } else {
 2099         ShouldNotReachHere();
 2100       }
 2101     } else if (cbuf) {
 2102       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2103       C2_MacroAssembler _masm(cbuf);
 2104       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2105       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2106         // stack->stack
 2107         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2108         if (ireg == Op_VecD) {
 2109           __ unspill(rscratch1, true, src_offset);
 2110           __ spill(rscratch1, true, dst_offset);
 2111         } else {
 2112           __ spill_copy128(src_offset, dst_offset);
 2113         }
 2114       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2115         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2116                ireg == Op_VecD ? __ T8B : __ T16B,
 2117                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2118       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2119         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2120                  ireg == Op_VecD ? __ D : __ Q,
 2121                  ra_->reg2offset(dst_lo));
 2122       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2123         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2124                    ireg == Op_VecD ? __ D : __ Q,
 2125                    ra_->reg2offset(src_lo));
 2126       } else {
 2127         ShouldNotReachHere();
 2128       }
 2129     }
 2130   } else if (cbuf) {
 2131     C2_MacroAssembler _masm(cbuf);
 2132     switch (src_lo_rc) {
 2133     case rc_int:
 2134       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2135         if (is64) {
 2136             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2137                    as_Register(Matcher::_regEncode[src_lo]));
 2138         } else {
 2139             C2_MacroAssembler _masm(cbuf);
 2140             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2141                     as_Register(Matcher::_regEncode[src_lo]));
 2142         }
 2143       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2144         if (is64) {
 2145             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2146                      as_Register(Matcher::_regEncode[src_lo]));
 2147         } else {
 2148             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2149                      as_Register(Matcher::_regEncode[src_lo]));
 2150         }
 2151       } else {                    // gpr --> stack spill
 2152         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2153         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2154       }
 2155       break;
 2156     case rc_float:
 2157       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2158         if (is64) {
 2159             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2160                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2161         } else {
 2162             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2163                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2164         }
 2165       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2166           if (cbuf) {
 2167             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2168                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2169         } else {
 2170             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2171                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2172         }
 2173       } else {                    // fpr --> stack spill
 2174         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2175         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2176                  is64 ? __ D : __ S, dst_offset);
 2177       }
 2178       break;
 2179     case rc_stack:
 2180       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2181         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2182       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2183         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2184                    is64 ? __ D : __ S, src_offset);
 2185       } else {                    // stack --> stack copy
 2186         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2187         __ unspill(rscratch1, is64, src_offset);
 2188         __ spill(rscratch1, is64, dst_offset);
 2189       }
 2190       break;
 2191     default:
 2192       assert(false, "bad rc_class for spill");
 2193       ShouldNotReachHere();
 2194     }
 2195   }
 2196 
 2197   if (st) {
 2198     st->print("spill ");
 2199     if (src_lo_rc == rc_stack) {
 2200       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2201     } else {
 2202       st->print("%s -> ", Matcher::regName[src_lo]);
 2203     }
 2204     if (dst_lo_rc == rc_stack) {
 2205       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2206     } else {
 2207       st->print("%s", Matcher::regName[dst_lo]);
 2208     }
 2209     if (bottom_type()->isa_vect() != NULL) {
 2210       int vsize = 0;
 2211       switch (ideal_reg()) {
 2212       case Op_VecD:
 2213         vsize = 64;
 2214         break;
 2215       case Op_VecX:
 2216         vsize = 128;
 2217         break;
 2218       case Op_VecA:
 2219         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2220         break;
 2221       default:
 2222         assert(false, "bad register type for spill");
 2223         ShouldNotReachHere();
 2224       }
 2225       st->print("\t# vector spill size = %d", vsize);
 2226     } else {
 2227       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2228     }
 2229   }
 2230 
 2231   return 0;
 2232 
 2233 }
 2234 
 2235 #ifndef PRODUCT
 2236 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2237   if (!ra_)
 2238     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2239   else
 2240     implementation(NULL, ra_, false, st);
 2241 }
 2242 #endif
 2243 
 2244 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2245   implementation(&cbuf, ra_, false, NULL);
 2246 }
 2247 
 2248 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2249   return MachNode::size(ra_);
 2250 }
 2251 
 2252 //=============================================================================
 2253 
 2254 #ifndef PRODUCT
 2255 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2256   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2257   int reg = ra_->get_reg_first(this);
 2258   st->print("add %s, rsp, #%d]\t# box lock",
 2259             Matcher::regName[reg], offset);
 2260 }
 2261 #endif
 2262 
 2263 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2264   C2_MacroAssembler _masm(&cbuf);
 2265 
 2266   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2267   int reg    = ra_->get_encode(this);
 2268 
 2269   // This add will handle any 24-bit signed offset. 24 bits allows an
 2270   // 8 megabyte stack frame.
 2271   __ add(as_Register(reg), sp, offset);
 2272 }
 2273 
 2274 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2275   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2276   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2277 
 2278   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2279     return NativeInstruction::instruction_size;
 2280   } else {
 2281     return 2 * NativeInstruction::instruction_size;
 2282   }
 2283 }
 2284 
 2285 //=============================================================================
 2286 
 2287 #ifndef PRODUCT
 2288 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2289 {
 2290   st->print_cr("# MachUEPNode");
 2291   if (UseCompressedClassPointers) {
 2292     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2293     if (CompressedKlassPointers::shift() != 0) {
 2294       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2295     }
 2296   } else {
 2297    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2298   }
 2299   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2300   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2301 }
 2302 #endif
 2303 
 2304 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2305 {
 2306   // This is the unverified entry point.
 2307   C2_MacroAssembler _masm(&cbuf);
 2308 
 2309   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2310   Label skip;
 2311   // TODO
 2312   // can we avoid this skip and still use a reloc?
 2313   __ br(Assembler::EQ, skip);
 2314   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2315   __ bind(skip);
 2316 }
 2317 
 2318 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2319 {
 2320   return MachNode::size(ra_);
 2321 }
 2322 
 2323 // REQUIRED EMIT CODE
 2324 
 2325 //=============================================================================
 2326 
 2327 // Emit exception handler code.
 2328 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2329 {
 2330   // mov rscratch1 #exception_blob_entry_point
 2331   // br rscratch1
 2332   // Note that the code buffer's insts_mark is always relative to insts.
 2333   // That's why we must use the macroassembler to generate a handler.
 2334   C2_MacroAssembler _masm(&cbuf);
 2335   address base = __ start_a_stub(size_exception_handler());
 2336   if (base == NULL) {
 2337     ciEnv::current()->record_failure("CodeCache is full");
 2338     return 0;  // CodeBuffer::expand failed
 2339   }
 2340   int offset = __ offset();
 2341   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2342   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2343   __ end_a_stub();
 2344   return offset;
 2345 }
 2346 
 2347 // Emit deopt handler code.
 2348 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2349 {
 2350   // Note that the code buffer's insts_mark is always relative to insts.
 2351   // That's why we must use the macroassembler to generate a handler.
 2352   C2_MacroAssembler _masm(&cbuf);
 2353   address base = __ start_a_stub(size_deopt_handler());
 2354   if (base == NULL) {
 2355     ciEnv::current()->record_failure("CodeCache is full");
 2356     return 0;  // CodeBuffer::expand failed
 2357   }
 2358   int offset = __ offset();
 2359 
 2360   __ adr(lr, __ pc());
 2361   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2362 
 2363   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2364   __ end_a_stub();
 2365   return offset;
 2366 }
 2367 
 2368 // REQUIRED MATCHER CODE
 2369 
 2370 //=============================================================================
 2371 
 2372 const bool Matcher::match_rule_supported(int opcode) {
 2373   if (!has_match_rule(opcode))
 2374     return false;
 2375 
 2376   bool ret_value = true;
 2377   switch (opcode) {
 2378     case Op_OnSpinWait:
 2379       return VM_Version::supports_on_spin_wait();
 2380     case Op_CacheWB:
 2381     case Op_CacheWBPreSync:
 2382     case Op_CacheWBPostSync:
 2383       if (!VM_Version::supports_data_cache_line_flush()) {
 2384         ret_value = false;
 2385       }
 2386       break;
 2387   }
 2388 
 2389   return ret_value; // Per default match rules are supported.
 2390 }
 2391 
 2392 // Identify extra cases that we might want to provide match rules for vector nodes and
 2393 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2394 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2395   if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) {
 2396     return false;
 2397   }
 2398   int bit_size = vlen * type2aelembytes(bt) * 8;
 2399   if (UseSVE == 0 && bit_size > 128) {
 2400     return false;
 2401   }
 2402   if (UseSVE > 0) {
 2403     return op_sve_supported(opcode);
 2404   } else { // NEON
 2405     // Special cases
 2406     switch (opcode) {
 2407     case Op_VectorMaskCmp:
 2408     // We don't have VectorReinterpret with bit_size less than 64 support for
 2409     // now, even for byte type. To be refined with fully VectorCast support.
 2410     case Op_VectorReinterpret:
 2411       if (vlen < 2 || bit_size < 64) {
 2412         return false;
 2413       }
 2414       break;
 2415     case Op_MulAddVS2VI:
 2416       if (bit_size < 128) {
 2417         return false;
 2418       }
 2419       break;
 2420     case Op_MulVL:
 2421       return false;
 2422     case Op_VectorLoadShuffle:
 2423     case Op_VectorRearrange:
 2424       if (vlen < 4) {
 2425         return false;
 2426       }
 2427       break;
 2428     // Some types of VectorCast are not implemented for now.
 2429     case Op_VectorCastI2X:
 2430       if (bt == T_BYTE) {
 2431         return false;
 2432       }
 2433       break;
 2434     case Op_VectorCastS2X:
 2435       if (vlen < 4 || bit_size < 64) {
 2436         return false;
 2437       }
 2438       break;
 2439     case Op_VectorCastF2X:
 2440     case Op_VectorCastD2X:
 2441       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2442         return false;
 2443       }
 2444       break;
 2445     default:
 2446       break;
 2447     }
 2448   }
 2449   return true; // Per default match rules are supported.
 2450 }
 2451 
 2452 const RegMask* Matcher::predicate_reg_mask(void) {
 2453   return &_PR_REG_mask;
 2454 }
 2455 
 2456 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2457   return new TypeVectMask(elemTy, length);
 2458 }
 2459 
 2460 // Vector calling convention not yet implemented.
 2461 const bool Matcher::supports_vector_calling_convention(void) {
 2462   return false;
 2463 }
 2464 
 2465 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2466   Unimplemented();
 2467   return OptoRegPair(0, 0);
 2468 }
 2469 
 2470 const int Matcher::float_pressure(int default_pressure_threshold) {
 2471   return default_pressure_threshold;
 2472 }
 2473 
 2474 // Is this branch offset short enough that a short branch can be used?
 2475 //
 2476 // NOTE: If the platform does not provide any short branch variants, then
 2477 //       this method should return false for offset 0.
 2478 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2479   // The passed offset is relative to address of the branch.
 2480 
 2481   return (-32768 <= offset && offset < 32768);
 2482 }
 2483 
 2484 // Vector width in bytes.
 2485 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2486   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2487   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2488   // Minimum 2 values in vector
 2489   if (size < 2*type2aelembytes(bt)) size = 0;
 2490   // But never < 4
 2491   if (size < 4) size = 0;
 2492   return size;
 2493 }
 2494 
 2495 // Limits on vector size (number of elements) loaded into vector.
 2496 const int Matcher::max_vector_size(const BasicType bt) {
 2497   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2498 }
 2499 const int Matcher::min_vector_size(const BasicType bt) {
 2500   int max_size = max_vector_size(bt);
 2501   if ((UseSVE > 0) && (MaxVectorSize >= 16)) {
 2502     // Currently vector length less than SVE vector register size is not supported.
 2503     return max_size;
 2504   } else { // NEON
 2505     // Limit the vector size to 8 bytes
 2506     int size = 8 / type2aelembytes(bt);
 2507     if (bt == T_BYTE) {
 2508       // To support vector api shuffle/rearrange.
 2509       size = 4;
 2510     } else if (bt == T_BOOLEAN) {
 2511       // To support vector api load/store mask.
 2512       size = 2;
 2513     }
 2514     if (size < 2) size = 2;
 2515     return MIN2(size,max_size);
 2516   }
 2517 }
 2518 
 2519 // Actual max scalable vector register length.
 2520 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2521   return Matcher::max_vector_size(bt);
 2522 }
 2523 
 2524 // Vector ideal reg.
 2525 const uint Matcher::vector_ideal_reg(int len) {
 2526   if (UseSVE > 0 && 16 <= len && len <= 256) {
 2527     return Op_VecA;
 2528   }
 2529   switch(len) {
 2530     // For 16-bit/32-bit mask vector, reuse VecD.
 2531     case  2:
 2532     case  4:
 2533     case  8: return Op_VecD;
 2534     case 16: return Op_VecX;
 2535   }
 2536   ShouldNotReachHere();
 2537   return 0;
 2538 }
 2539 
 2540 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2541   ShouldNotReachHere(); // generic vector operands not supported
 2542   return NULL;
 2543 }
 2544 
 2545 bool Matcher::is_generic_reg2reg_move(MachNode* m) {
 2546   ShouldNotReachHere();  // generic vector operands not supported
 2547   return false;
 2548 }
 2549 
 2550 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2551   ShouldNotReachHere();  // generic vector operands not supported
 2552   return false;
 2553 }
 2554 
 2555 // Return whether or not this register is ever used as an argument.
 2556 // This function is used on startup to build the trampoline stubs in
 2557 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2558 // call in the trampoline, and arguments in those registers not be
 2559 // available to the callee.
 2560 bool Matcher::can_be_java_arg(int reg)
 2561 {
 2562   return
 2563     reg ==  R0_num || reg == R0_H_num ||
 2564     reg ==  R1_num || reg == R1_H_num ||
 2565     reg ==  R2_num || reg == R2_H_num ||
 2566     reg ==  R3_num || reg == R3_H_num ||
 2567     reg ==  R4_num || reg == R4_H_num ||
 2568     reg ==  R5_num || reg == R5_H_num ||
 2569     reg ==  R6_num || reg == R6_H_num ||
 2570     reg ==  R7_num || reg == R7_H_num ||
 2571     reg ==  V0_num || reg == V0_H_num ||
 2572     reg ==  V1_num || reg == V1_H_num ||
 2573     reg ==  V2_num || reg == V2_H_num ||
 2574     reg ==  V3_num || reg == V3_H_num ||
 2575     reg ==  V4_num || reg == V4_H_num ||
 2576     reg ==  V5_num || reg == V5_H_num ||
 2577     reg ==  V6_num || reg == V6_H_num ||
 2578     reg ==  V7_num || reg == V7_H_num;
 2579 }
 2580 
 2581 bool Matcher::is_spillable_arg(int reg)
 2582 {
 2583   return can_be_java_arg(reg);
 2584 }
 2585 
 2586 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2587   return false;
 2588 }
 2589 
 2590 RegMask Matcher::divI_proj_mask() {
 2591   ShouldNotReachHere();
 2592   return RegMask();
 2593 }
 2594 
 2595 // Register for MODI projection of divmodI.
 2596 RegMask Matcher::modI_proj_mask() {
 2597   ShouldNotReachHere();
 2598   return RegMask();
 2599 }
 2600 
 2601 // Register for DIVL projection of divmodL.
 2602 RegMask Matcher::divL_proj_mask() {
 2603   ShouldNotReachHere();
 2604   return RegMask();
 2605 }
 2606 
 2607 // Register for MODL projection of divmodL.
 2608 RegMask Matcher::modL_proj_mask() {
 2609   ShouldNotReachHere();
 2610   return RegMask();
 2611 }
 2612 
 2613 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2614   return FP_REG_mask();
 2615 }
 2616 
 2617 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2618   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2619     Node* u = addp->fast_out(i);
 2620     if (u->is_LoadStore()) {
 2621       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2622       // instructions) only take register indirect as an operand, so
 2623       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2624       // must fail.
 2625       return false;
 2626     }
 2627     if (u->is_Mem()) {
 2628       int opsize = u->as_Mem()->memory_size();
 2629       assert(opsize > 0, "unexpected memory operand size");
 2630       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2631         return false;
 2632       }
 2633     }
 2634   }
 2635   return true;
 2636 }
 2637 
 2638 // Should the matcher clone input 'm' of node 'n'?
 2639 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2640   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
 2641     mstack.push(m, Visit);           // m = ShiftCntV
 2642     return true;
 2643   }
 2644   return false;
 2645 }
 2646 
 2647 // Should the Matcher clone shifts on addressing modes, expecting them
 2648 // to be subsumed into complex addressing expressions or compute them
 2649 // into registers?
 2650 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2651   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2652     return true;
 2653   }
 2654 
 2655   Node *off = m->in(AddPNode::Offset);
 2656   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2657       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2658       // Are there other uses besides address expressions?
 2659       !is_visited(off)) {
 2660     address_visited.set(off->_idx); // Flag as address_visited
 2661     mstack.push(off->in(2), Visit);
 2662     Node *conv = off->in(1);
 2663     if (conv->Opcode() == Op_ConvI2L &&
 2664         // Are there other uses besides address expressions?
 2665         !is_visited(conv)) {
 2666       address_visited.set(conv->_idx); // Flag as address_visited
 2667       mstack.push(conv->in(1), Pre_Visit);
 2668     } else {
 2669       mstack.push(conv, Pre_Visit);
 2670     }
 2671     address_visited.test_set(m->_idx); // Flag as address_visited
 2672     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2673     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2674     return true;
 2675   } else if (off->Opcode() == Op_ConvI2L &&
 2676              // Are there other uses besides address expressions?
 2677              !is_visited(off)) {
 2678     address_visited.test_set(m->_idx); // Flag as address_visited
 2679     address_visited.set(off->_idx); // Flag as address_visited
 2680     mstack.push(off->in(1), Pre_Visit);
 2681     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2682     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2683     return true;
 2684   }
 2685   return false;
 2686 }
 2687 
 2688 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2689   C2_MacroAssembler _masm(&cbuf);                                       \
 2690   {                                                                     \
 2691     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2692     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2693     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2694     __ INSN(REG, as_Register(BASE));                                    \
 2695   }
 2696 
 2697 
 2698 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2699   {
 2700     Address::extend scale;
 2701 
 2702     // Hooboy, this is fugly.  We need a way to communicate to the
 2703     // encoder that the index needs to be sign extended, so we have to
 2704     // enumerate all the cases.
 2705     switch (opcode) {
 2706     case INDINDEXSCALEDI2L:
 2707     case INDINDEXSCALEDI2LN:
 2708     case INDINDEXI2L:
 2709     case INDINDEXI2LN:
 2710       scale = Address::sxtw(size);
 2711       break;
 2712     default:
 2713       scale = Address::lsl(size);
 2714     }
 2715 
 2716     if (index == -1) {
 2717       return Address(base, disp);
 2718     } else {
 2719       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2720       return Address(base, as_Register(index), scale);
 2721     }
 2722   }
 2723 
 2724 
 2725 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2726 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2727 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2728 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2729                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2730 
 2731   // Used for all non-volatile memory accesses.  The use of
 2732   // $mem->opcode() to discover whether this pattern uses sign-extended
 2733   // offsets is something of a kludge.
 2734   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2735                         Register reg, int opcode,
 2736                         Register base, int index, int scale, int disp,
 2737                         int size_in_memory)
 2738   {
 2739     Address addr = mem2address(opcode, base, index, scale, disp);
 2740     if (addr.getMode() == Address::base_plus_offset) {
 2741       /* If we get an out-of-range offset it is a bug in the compiler,
 2742          so we assert here. */
 2743       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2744              "c2 compiler bug");
 2745       /* Fix up any out-of-range offsets. */
 2746       assert_different_registers(rscratch1, base);
 2747       assert_different_registers(rscratch1, reg);
 2748       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2749     }
 2750     (masm.*insn)(reg, addr);
 2751   }
 2752 
 2753   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2754                         FloatRegister reg, int opcode,
 2755                         Register base, int index, int size, int disp,
 2756                         int size_in_memory)
 2757   {
 2758     Address::extend scale;
 2759 
 2760     switch (opcode) {
 2761     case INDINDEXSCALEDI2L:
 2762     case INDINDEXSCALEDI2LN:
 2763       scale = Address::sxtw(size);
 2764       break;
 2765     default:
 2766       scale = Address::lsl(size);
 2767     }
 2768 
 2769     if (index == -1) {
 2770       /* If we get an out-of-range offset it is a bug in the compiler,
 2771          so we assert here. */
 2772       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2773       /* Fix up any out-of-range offsets. */
 2774       assert_different_registers(rscratch1, base);
 2775       Address addr = Address(base, disp);
 2776       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2777       (masm.*insn)(reg, addr);
 2778     } else {
 2779       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2780       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2781     }
 2782   }
 2783 
 2784   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2785                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2786                         int opcode, Register base, int index, int size, int disp)
 2787   {
 2788     if (index == -1) {
 2789       (masm.*insn)(reg, T, Address(base, disp));
 2790     } else {
 2791       assert(disp == 0, "unsupported address mode");
 2792       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2793     }
 2794   }
 2795 
 2796 %}
 2797 
 2798 
 2799 
 2800 //----------ENCODING BLOCK-----------------------------------------------------
 2801 // This block specifies the encoding classes used by the compiler to
 2802 // output byte streams.  Encoding classes are parameterized macros
 2803 // used by Machine Instruction Nodes in order to generate the bit
 2804 // encoding of the instruction.  Operands specify their base encoding
 2805 // interface with the interface keyword.  There are currently
 2806 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2807 // COND_INTER.  REG_INTER causes an operand to generate a function
 2808 // which returns its register number when queried.  CONST_INTER causes
 2809 // an operand to generate a function which returns the value of the
 2810 // constant when queried.  MEMORY_INTER causes an operand to generate
 2811 // four functions which return the Base Register, the Index Register,
 2812 // the Scale Value, and the Offset Value of the operand when queried.
 2813 // COND_INTER causes an operand to generate six functions which return
 2814 // the encoding code (ie - encoding bits for the instruction)
 2815 // associated with each basic boolean condition for a conditional
 2816 // instruction.
 2817 //
 2818 // Instructions specify two basic values for encoding.  Again, a
 2819 // function is available to check if the constant displacement is an
 2820 // oop. They use the ins_encode keyword to specify their encoding
 2821 // classes (which must be a sequence of enc_class names, and their
 2822 // parameters, specified in the encoding block), and they use the
 2823 // opcode keyword to specify, in order, their primary, secondary, and
 2824 // tertiary opcode.  Only the opcode sections which a particular
 2825 // instruction needs for encoding need to be specified.
 2826 encode %{
 2827   // Build emit functions for each basic byte or larger field in the
 2828   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2829   // from C++ code in the enc_class source block.  Emit functions will
 2830   // live in the main source block for now.  In future, we can
 2831   // generalize this by adding a syntax that specifies the sizes of
 2832   // fields in an order, so that the adlc can build the emit functions
 2833   // automagically
 2834 
 2835   // catch all for unimplemented encodings
 2836   enc_class enc_unimplemented %{
 2837     C2_MacroAssembler _masm(&cbuf);
 2838     __ unimplemented("C2 catch all");
 2839   %}
 2840 
 2841   // BEGIN Non-volatile memory access
 2842 
 2843   // This encoding class is generated automatically from ad_encode.m4.
 2844   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2845   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2846     Register dst_reg = as_Register($dst$$reg);
 2847     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2848                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2849   %}
 2850 
 2851   // This encoding class is generated automatically from ad_encode.m4.
 2852   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2853   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2854     Register dst_reg = as_Register($dst$$reg);
 2855     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2856                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2857   %}
 2858 
 2859   // This encoding class is generated automatically from ad_encode.m4.
 2860   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2861   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2862     Register dst_reg = as_Register($dst$$reg);
 2863     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2864                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2865   %}
 2866 
 2867   // This encoding class is generated automatically from ad_encode.m4.
 2868   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2869   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2870     Register dst_reg = as_Register($dst$$reg);
 2871     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2872                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2873   %}
 2874 
 2875   // This encoding class is generated automatically from ad_encode.m4.
 2876   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2877   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2878     Register dst_reg = as_Register($dst$$reg);
 2879     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2880                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2881   %}
 2882 
 2883   // This encoding class is generated automatically from ad_encode.m4.
 2884   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2885   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2886     Register dst_reg = as_Register($dst$$reg);
 2887     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2888                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2889   %}
 2890 
 2891   // This encoding class is generated automatically from ad_encode.m4.
 2892   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2893   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2894     Register dst_reg = as_Register($dst$$reg);
 2895     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2896                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2897   %}
 2898 
 2899   // This encoding class is generated automatically from ad_encode.m4.
 2900   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2901   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2902     Register dst_reg = as_Register($dst$$reg);
 2903     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2904                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2905   %}
 2906 
 2907   // This encoding class is generated automatically from ad_encode.m4.
 2908   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2909   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2910     Register dst_reg = as_Register($dst$$reg);
 2911     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2912                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2913   %}
 2914 
 2915   // This encoding class is generated automatically from ad_encode.m4.
 2916   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2917   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2918     Register dst_reg = as_Register($dst$$reg);
 2919     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2921   %}
 2922 
 2923   // This encoding class is generated automatically from ad_encode.m4.
 2924   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2925   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2926     Register dst_reg = as_Register($dst$$reg);
 2927     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2928                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2929   %}
 2930 
 2931   // This encoding class is generated automatically from ad_encode.m4.
 2932   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2933   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2934     Register dst_reg = as_Register($dst$$reg);
 2935     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2936                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2937   %}
 2938 
 2939   // This encoding class is generated automatically from ad_encode.m4.
 2940   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2941   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2942     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2943     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2945   %}
 2946 
 2947   // This encoding class is generated automatically from ad_encode.m4.
 2948   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2949   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2950     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2951     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2952                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2953   %}
 2954 
 2955   // This encoding class is generated automatically from ad_encode.m4.
 2956   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2957   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2958     Register src_reg = as_Register($src$$reg);
 2959     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2960                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2961   %}
 2962 
 2963   // This encoding class is generated automatically from ad_encode.m4.
 2964   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2965   enc_class aarch64_enc_strb0(memory1 mem) %{
 2966     C2_MacroAssembler _masm(&cbuf);
 2967     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2968                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2969   %}
 2970 
 2971   // This encoding class is generated automatically from ad_encode.m4.
 2972   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2973   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2974     Register src_reg = as_Register($src$$reg);
 2975     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2976                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2977   %}
 2978 
 2979   // This encoding class is generated automatically from ad_encode.m4.
 2980   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2981   enc_class aarch64_enc_strh0(memory2 mem) %{
 2982     C2_MacroAssembler _masm(&cbuf);
 2983     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2984                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2985   %}
 2986 
 2987   // This encoding class is generated automatically from ad_encode.m4.
 2988   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2989   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2990     Register src_reg = as_Register($src$$reg);
 2991     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2992                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2993   %}
 2994 
 2995   // This encoding class is generated automatically from ad_encode.m4.
 2996   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2997   enc_class aarch64_enc_strw0(memory4 mem) %{
 2998     C2_MacroAssembler _masm(&cbuf);
 2999     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3000                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3001   %}
 3002 
 3003   // This encoding class is generated automatically from ad_encode.m4.
 3004   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3005   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3006     Register src_reg = as_Register($src$$reg);
 3007     // we sometimes get asked to store the stack pointer into the
 3008     // current thread -- we cannot do that directly on AArch64
 3009     if (src_reg == r31_sp) {
 3010       C2_MacroAssembler _masm(&cbuf);
 3011       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3012       __ mov(rscratch2, sp);
 3013       src_reg = rscratch2;
 3014     }
 3015     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3016                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3017   %}
 3018 
 3019   // This encoding class is generated automatically from ad_encode.m4.
 3020   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3021   enc_class aarch64_enc_str0(memory8 mem) %{
 3022     C2_MacroAssembler _masm(&cbuf);
 3023     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3024                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3025   %}
 3026 
 3027   // This encoding class is generated automatically from ad_encode.m4.
 3028   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3029   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3030     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3031     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3032                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3033   %}
 3034 
 3035   // This encoding class is generated automatically from ad_encode.m4.
 3036   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3037   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3038     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3039     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3041   %}
 3042 
 3043   // This encoding class is generated automatically from ad_encode.m4.
 3044   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3045   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3046       C2_MacroAssembler _masm(&cbuf);
 3047       __ membar(Assembler::StoreStore);
 3048       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3049                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3050   %}
 3051 
 3052   // END Non-volatile memory access
 3053 
 3054   // Vector loads and stores
 3055   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3056     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3057     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3058        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3059   %}
 3060 
 3061   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3062     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3063     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3064        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3065   %}
 3066 
 3067   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3068     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3069     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3070        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3071   %}
 3072 
 3073   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3074     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3075     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3076        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3077   %}
 3078 
 3079   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3080     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3081     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3082        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3083   %}
 3084 
 3085   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3086     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3087     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3088        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3089   %}
 3090 
 3091   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3092     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3093     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3094        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3095   %}
 3096 
 3097   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3098     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3099     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3100        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3101   %}
 3102 
 3103   // volatile loads and stores
 3104 
 3105   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3106     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3107                  rscratch1, stlrb);
 3108   %}
 3109 
 3110   enc_class aarch64_enc_stlrb0(memory mem) %{
 3111     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3112                  rscratch1, stlrb);
 3113   %}
 3114 
 3115   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3116     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3117                  rscratch1, stlrh);
 3118   %}
 3119 
 3120   enc_class aarch64_enc_stlrh0(memory mem) %{
 3121     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3122                  rscratch1, stlrh);
 3123   %}
 3124 
 3125   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3126     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3127                  rscratch1, stlrw);
 3128   %}
 3129 
 3130   enc_class aarch64_enc_stlrw0(memory mem) %{
 3131     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3132                  rscratch1, stlrw);
 3133   %}
 3134 
 3135   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3136     Register dst_reg = as_Register($dst$$reg);
 3137     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3138              rscratch1, ldarb);
 3139     __ sxtbw(dst_reg, dst_reg);
 3140   %}
 3141 
 3142   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3143     Register dst_reg = as_Register($dst$$reg);
 3144     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3145              rscratch1, ldarb);
 3146     __ sxtb(dst_reg, dst_reg);
 3147   %}
 3148 
 3149   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3150     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3151              rscratch1, ldarb);
 3152   %}
 3153 
 3154   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3155     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3156              rscratch1, ldarb);
 3157   %}
 3158 
 3159   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3160     Register dst_reg = as_Register($dst$$reg);
 3161     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3162              rscratch1, ldarh);
 3163     __ sxthw(dst_reg, dst_reg);
 3164   %}
 3165 
 3166   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3167     Register dst_reg = as_Register($dst$$reg);
 3168     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3169              rscratch1, ldarh);
 3170     __ sxth(dst_reg, dst_reg);
 3171   %}
 3172 
 3173   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3174     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3175              rscratch1, ldarh);
 3176   %}
 3177 
 3178   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3179     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3180              rscratch1, ldarh);
 3181   %}
 3182 
 3183   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3184     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3185              rscratch1, ldarw);
 3186   %}
 3187 
 3188   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3189     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3190              rscratch1, ldarw);
 3191   %}
 3192 
 3193   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3194     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3195              rscratch1, ldar);
 3196   %}
 3197 
 3198   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3199     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3200              rscratch1, ldarw);
 3201     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3202   %}
 3203 
 3204   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3205     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3206              rscratch1, ldar);
 3207     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3208   %}
 3209 
 3210   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3211     Register src_reg = as_Register($src$$reg);
 3212     // we sometimes get asked to store the stack pointer into the
 3213     // current thread -- we cannot do that directly on AArch64
 3214     if (src_reg == r31_sp) {
 3215       C2_MacroAssembler _masm(&cbuf);
 3216       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3217       __ mov(rscratch2, sp);
 3218       src_reg = rscratch2;
 3219     }
 3220     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3221                  rscratch1, stlr);
 3222   %}
 3223 
 3224   enc_class aarch64_enc_stlr0(memory mem) %{
 3225     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3226                  rscratch1, stlr);
 3227   %}
 3228 
 3229   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3230     {
 3231       C2_MacroAssembler _masm(&cbuf);
 3232       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3233       __ fmovs(rscratch2, src_reg);
 3234     }
 3235     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3236                  rscratch1, stlrw);
 3237   %}
 3238 
 3239   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3240     {
 3241       C2_MacroAssembler _masm(&cbuf);
 3242       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3243       __ fmovd(rscratch2, src_reg);
 3244     }
 3245     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3246                  rscratch1, stlr);
 3247   %}
 3248 
 3249   // synchronized read/update encodings
 3250 
 3251   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3252     C2_MacroAssembler _masm(&cbuf);
 3253     Register dst_reg = as_Register($dst$$reg);
 3254     Register base = as_Register($mem$$base);
 3255     int index = $mem$$index;
 3256     int scale = $mem$$scale;
 3257     int disp = $mem$$disp;
 3258     if (index == -1) {
 3259        if (disp != 0) {
 3260         __ lea(rscratch1, Address(base, disp));
 3261         __ ldaxr(dst_reg, rscratch1);
 3262       } else {
 3263         // TODO
 3264         // should we ever get anything other than this case?
 3265         __ ldaxr(dst_reg, base);
 3266       }
 3267     } else {
 3268       Register index_reg = as_Register(index);
 3269       if (disp == 0) {
 3270         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3271         __ ldaxr(dst_reg, rscratch1);
 3272       } else {
 3273         __ lea(rscratch1, Address(base, disp));
 3274         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3275         __ ldaxr(dst_reg, rscratch1);
 3276       }
 3277     }
 3278   %}
 3279 
 3280   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3281     C2_MacroAssembler _masm(&cbuf);
 3282     Register src_reg = as_Register($src$$reg);
 3283     Register base = as_Register($mem$$base);
 3284     int index = $mem$$index;
 3285     int scale = $mem$$scale;
 3286     int disp = $mem$$disp;
 3287     if (index == -1) {
 3288        if (disp != 0) {
 3289         __ lea(rscratch2, Address(base, disp));
 3290         __ stlxr(rscratch1, src_reg, rscratch2);
 3291       } else {
 3292         // TODO
 3293         // should we ever get anything other than this case?
 3294         __ stlxr(rscratch1, src_reg, base);
 3295       }
 3296     } else {
 3297       Register index_reg = as_Register(index);
 3298       if (disp == 0) {
 3299         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3300         __ stlxr(rscratch1, src_reg, rscratch2);
 3301       } else {
 3302         __ lea(rscratch2, Address(base, disp));
 3303         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3304         __ stlxr(rscratch1, src_reg, rscratch2);
 3305       }
 3306     }
 3307     __ cmpw(rscratch1, zr);
 3308   %}
 3309 
 3310   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3311     C2_MacroAssembler _masm(&cbuf);
 3312     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3313     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3314                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3315                /*weak*/ false, noreg);
 3316   %}
 3317 
 3318   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3319     C2_MacroAssembler _masm(&cbuf);
 3320     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3321     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3322                Assembler::word, /*acquire*/ false, /*release*/ true,
 3323                /*weak*/ false, noreg);
 3324   %}
 3325 
 3326   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3327     C2_MacroAssembler _masm(&cbuf);
 3328     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3329     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3330                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3331                /*weak*/ false, noreg);
 3332   %}
 3333 
 3334   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3335     C2_MacroAssembler _masm(&cbuf);
 3336     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3337     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3338                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3339                /*weak*/ false, noreg);
 3340   %}
 3341 
 3342 
 3343   // The only difference between aarch64_enc_cmpxchg and
 3344   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3345   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3346   // lock.
 3347   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3348     C2_MacroAssembler _masm(&cbuf);
 3349     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3350     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3351                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3352                /*weak*/ false, noreg);
 3353   %}
 3354 
 3355   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3356     C2_MacroAssembler _masm(&cbuf);
 3357     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3358     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3359                Assembler::word, /*acquire*/ true, /*release*/ true,
 3360                /*weak*/ false, noreg);
 3361   %}
 3362 
 3363   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3364     C2_MacroAssembler _masm(&cbuf);
 3365     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3366     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3367                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3368                /*weak*/ false, noreg);
 3369   %}
 3370 
 3371   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3372     C2_MacroAssembler _masm(&cbuf);
 3373     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3374     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3375                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3376                /*weak*/ false, noreg);
 3377   %}
 3378 
 3379   // auxiliary used for CompareAndSwapX to set result register
 3380   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3381     C2_MacroAssembler _masm(&cbuf);
 3382     Register res_reg = as_Register($res$$reg);
 3383     __ cset(res_reg, Assembler::EQ);
 3384   %}
 3385 
 3386   // prefetch encodings
 3387 
 3388   enc_class aarch64_enc_prefetchw(memory mem) %{
 3389     C2_MacroAssembler _masm(&cbuf);
 3390     Register base = as_Register($mem$$base);
 3391     int index = $mem$$index;
 3392     int scale = $mem$$scale;
 3393     int disp = $mem$$disp;
 3394     if (index == -1) {
 3395       __ prfm(Address(base, disp), PSTL1KEEP);
 3396     } else {
 3397       Register index_reg = as_Register(index);
 3398       if (disp == 0) {
 3399         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3400       } else {
 3401         __ lea(rscratch1, Address(base, disp));
 3402 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3403       }
 3404     }
 3405   %}
 3406 
 3407   /// mov envcodings
 3408 
 3409   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3410     C2_MacroAssembler _masm(&cbuf);
 3411     uint32_t con = (uint32_t)$src$$constant;
 3412     Register dst_reg = as_Register($dst$$reg);
 3413     if (con == 0) {
 3414       __ movw(dst_reg, zr);
 3415     } else {
 3416       __ movw(dst_reg, con);
 3417     }
 3418   %}
 3419 
 3420   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3421     C2_MacroAssembler _masm(&cbuf);
 3422     Register dst_reg = as_Register($dst$$reg);
 3423     uint64_t con = (uint64_t)$src$$constant;
 3424     if (con == 0) {
 3425       __ mov(dst_reg, zr);
 3426     } else {
 3427       __ mov(dst_reg, con);
 3428     }
 3429   %}
 3430 
 3431   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3432     C2_MacroAssembler _masm(&cbuf);
 3433     Register dst_reg = as_Register($dst$$reg);
 3434     address con = (address)$src$$constant;
 3435     if (con == NULL || con == (address)1) {
 3436       ShouldNotReachHere();
 3437     } else {
 3438       relocInfo::relocType rtype = $src->constant_reloc();
 3439       if (rtype == relocInfo::oop_type) {
 3440         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3441       } else if (rtype == relocInfo::metadata_type) {
 3442         __ mov_metadata(dst_reg, (Metadata*)con);
 3443       } else {
 3444         assert(rtype == relocInfo::none, "unexpected reloc type");
 3445         if (! __ is_valid_AArch64_address(con) ||
 3446             con < (address)(uintptr_t)os::vm_page_size()) {
 3447           __ mov(dst_reg, con);
 3448         } else {
 3449           uint64_t offset;
 3450           __ adrp(dst_reg, con, offset);
 3451           __ add(dst_reg, dst_reg, offset);
 3452         }
 3453       }
 3454     }
 3455   %}
 3456 
 3457   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3458     C2_MacroAssembler _masm(&cbuf);
 3459     Register dst_reg = as_Register($dst$$reg);
 3460     __ mov(dst_reg, zr);
 3461   %}
 3462 
 3463   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3464     C2_MacroAssembler _masm(&cbuf);
 3465     Register dst_reg = as_Register($dst$$reg);
 3466     __ mov(dst_reg, (uint64_t)1);
 3467   %}
 3468 
 3469   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3470     C2_MacroAssembler _masm(&cbuf);
 3471     __ load_byte_map_base($dst$$Register);
 3472   %}
 3473 
 3474   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3475     C2_MacroAssembler _masm(&cbuf);
 3476     Register dst_reg = as_Register($dst$$reg);
 3477     address con = (address)$src$$constant;
 3478     if (con == NULL) {
 3479       ShouldNotReachHere();
 3480     } else {
 3481       relocInfo::relocType rtype = $src->constant_reloc();
 3482       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3483       __ set_narrow_oop(dst_reg, (jobject)con);
 3484     }
 3485   %}
 3486 
 3487   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3488     C2_MacroAssembler _masm(&cbuf);
 3489     Register dst_reg = as_Register($dst$$reg);
 3490     __ mov(dst_reg, zr);
 3491   %}
 3492 
 3493   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3494     C2_MacroAssembler _masm(&cbuf);
 3495     Register dst_reg = as_Register($dst$$reg);
 3496     address con = (address)$src$$constant;
 3497     if (con == NULL) {
 3498       ShouldNotReachHere();
 3499     } else {
 3500       relocInfo::relocType rtype = $src->constant_reloc();
 3501       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3502       __ set_narrow_klass(dst_reg, (Klass *)con);
 3503     }
 3504   %}
 3505 
 3506   // arithmetic encodings
 3507 
 3508   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3509     C2_MacroAssembler _masm(&cbuf);
 3510     Register dst_reg = as_Register($dst$$reg);
 3511     Register src_reg = as_Register($src1$$reg);
 3512     int32_t con = (int32_t)$src2$$constant;
 3513     // add has primary == 0, subtract has primary == 1
 3514     if ($primary) { con = -con; }
 3515     if (con < 0) {
 3516       __ subw(dst_reg, src_reg, -con);
 3517     } else {
 3518       __ addw(dst_reg, src_reg, con);
 3519     }
 3520   %}
 3521 
 3522   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3523     C2_MacroAssembler _masm(&cbuf);
 3524     Register dst_reg = as_Register($dst$$reg);
 3525     Register src_reg = as_Register($src1$$reg);
 3526     int32_t con = (int32_t)$src2$$constant;
 3527     // add has primary == 0, subtract has primary == 1
 3528     if ($primary) { con = -con; }
 3529     if (con < 0) {
 3530       __ sub(dst_reg, src_reg, -con);
 3531     } else {
 3532       __ add(dst_reg, src_reg, con);
 3533     }
 3534   %}
 3535 
 3536   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3537     C2_MacroAssembler _masm(&cbuf);
 3538    Register dst_reg = as_Register($dst$$reg);
 3539    Register src1_reg = as_Register($src1$$reg);
 3540    Register src2_reg = as_Register($src2$$reg);
 3541     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3542   %}
 3543 
 3544   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3545     C2_MacroAssembler _masm(&cbuf);
 3546    Register dst_reg = as_Register($dst$$reg);
 3547    Register src1_reg = as_Register($src1$$reg);
 3548    Register src2_reg = as_Register($src2$$reg);
 3549     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3550   %}
 3551 
 3552   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3553     C2_MacroAssembler _masm(&cbuf);
 3554    Register dst_reg = as_Register($dst$$reg);
 3555    Register src1_reg = as_Register($src1$$reg);
 3556    Register src2_reg = as_Register($src2$$reg);
 3557     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3558   %}
 3559 
 3560   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3561     C2_MacroAssembler _masm(&cbuf);
 3562    Register dst_reg = as_Register($dst$$reg);
 3563    Register src1_reg = as_Register($src1$$reg);
 3564    Register src2_reg = as_Register($src2$$reg);
 3565     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3566   %}
 3567 
 3568   // compare instruction encodings
 3569 
 3570   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3571     C2_MacroAssembler _masm(&cbuf);
 3572     Register reg1 = as_Register($src1$$reg);
 3573     Register reg2 = as_Register($src2$$reg);
 3574     __ cmpw(reg1, reg2);
 3575   %}
 3576 
 3577   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3578     C2_MacroAssembler _masm(&cbuf);
 3579     Register reg = as_Register($src1$$reg);
 3580     int32_t val = $src2$$constant;
 3581     if (val >= 0) {
 3582       __ subsw(zr, reg, val);
 3583     } else {
 3584       __ addsw(zr, reg, -val);
 3585     }
 3586   %}
 3587 
 3588   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3589     C2_MacroAssembler _masm(&cbuf);
 3590     Register reg1 = as_Register($src1$$reg);
 3591     uint32_t val = (uint32_t)$src2$$constant;
 3592     __ movw(rscratch1, val);
 3593     __ cmpw(reg1, rscratch1);
 3594   %}
 3595 
 3596   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3597     C2_MacroAssembler _masm(&cbuf);
 3598     Register reg1 = as_Register($src1$$reg);
 3599     Register reg2 = as_Register($src2$$reg);
 3600     __ cmp(reg1, reg2);
 3601   %}
 3602 
 3603   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3604     C2_MacroAssembler _masm(&cbuf);
 3605     Register reg = as_Register($src1$$reg);
 3606     int64_t val = $src2$$constant;
 3607     if (val >= 0) {
 3608       __ subs(zr, reg, val);
 3609     } else if (val != -val) {
 3610       __ adds(zr, reg, -val);
 3611     } else {
 3612     // aargh, Long.MIN_VALUE is a special case
 3613       __ orr(rscratch1, zr, (uint64_t)val);
 3614       __ subs(zr, reg, rscratch1);
 3615     }
 3616   %}
 3617 
 3618   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3619     C2_MacroAssembler _masm(&cbuf);
 3620     Register reg1 = as_Register($src1$$reg);
 3621     uint64_t val = (uint64_t)$src2$$constant;
 3622     __ mov(rscratch1, val);
 3623     __ cmp(reg1, rscratch1);
 3624   %}
 3625 
 3626   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3627     C2_MacroAssembler _masm(&cbuf);
 3628     Register reg1 = as_Register($src1$$reg);
 3629     Register reg2 = as_Register($src2$$reg);
 3630     __ cmp(reg1, reg2);
 3631   %}
 3632 
 3633   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3634     C2_MacroAssembler _masm(&cbuf);
 3635     Register reg1 = as_Register($src1$$reg);
 3636     Register reg2 = as_Register($src2$$reg);
 3637     __ cmpw(reg1, reg2);
 3638   %}
 3639 
 3640   enc_class aarch64_enc_testp(iRegP src) %{
 3641     C2_MacroAssembler _masm(&cbuf);
 3642     Register reg = as_Register($src$$reg);
 3643     __ cmp(reg, zr);
 3644   %}
 3645 
 3646   enc_class aarch64_enc_testn(iRegN src) %{
 3647     C2_MacroAssembler _masm(&cbuf);
 3648     Register reg = as_Register($src$$reg);
 3649     __ cmpw(reg, zr);
 3650   %}
 3651 
 3652   enc_class aarch64_enc_b(label lbl) %{
 3653     C2_MacroAssembler _masm(&cbuf);
 3654     Label *L = $lbl$$label;
 3655     __ b(*L);
 3656   %}
 3657 
 3658   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3659     C2_MacroAssembler _masm(&cbuf);
 3660     Label *L = $lbl$$label;
 3661     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3662   %}
 3663 
 3664   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3665     C2_MacroAssembler _masm(&cbuf);
 3666     Label *L = $lbl$$label;
 3667     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3668   %}
 3669 
 3670   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3671   %{
 3672      Register sub_reg = as_Register($sub$$reg);
 3673      Register super_reg = as_Register($super$$reg);
 3674      Register temp_reg = as_Register($temp$$reg);
 3675      Register result_reg = as_Register($result$$reg);
 3676 
 3677      Label miss;
 3678      C2_MacroAssembler _masm(&cbuf);
 3679      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3680                                      NULL, &miss,
 3681                                      /*set_cond_codes:*/ true);
 3682      if ($primary) {
 3683        __ mov(result_reg, zr);
 3684      }
 3685      __ bind(miss);
 3686   %}
 3687 
 3688   enc_class aarch64_enc_java_static_call(method meth) %{
 3689     C2_MacroAssembler _masm(&cbuf);
 3690 
 3691     address addr = (address)$meth$$method;
 3692     address call;
 3693     if (!_method) {
 3694       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3695       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3696       if (call == NULL) {
 3697         ciEnv::current()->record_failure("CodeCache is full");
 3698         return;
 3699       }
 3700     } else {
 3701       int method_index = resolved_method_index(cbuf);
 3702       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3703                                                   : static_call_Relocation::spec(method_index);
 3704       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3705       if (call == NULL) {
 3706         ciEnv::current()->record_failure("CodeCache is full");
 3707         return;
 3708       }
 3709       // Emit stub for static call
 3710       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3711       if (stub == NULL) {
 3712         ciEnv::current()->record_failure("CodeCache is full");
 3713         return;
 3714       }
 3715     }
 3716 
 3717     // Only non uncommon_trap calls need to reinitialize ptrue.
 3718     if (Compile::current()->max_vector_size() >= 16 && uncommon_trap_request() == 0) {
 3719       __ reinitialize_ptrue();
 3720     }
 3721   %}
 3722 
 3723   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3724     C2_MacroAssembler _masm(&cbuf);
 3725     int method_index = resolved_method_index(cbuf);
 3726     address call = __ ic_call((address)$meth$$method, method_index);
 3727     if (call == NULL) {
 3728       ciEnv::current()->record_failure("CodeCache is full");
 3729       return;
 3730     } else if (Compile::current()->max_vector_size() >= 16) {
 3731       __ reinitialize_ptrue();
 3732     }
 3733   %}
 3734 
 3735   enc_class aarch64_enc_call_epilog() %{
 3736     C2_MacroAssembler _masm(&cbuf);
 3737     if (VerifyStackAtCalls) {
 3738       // Check that stack depth is unchanged: find majik cookie on stack
 3739       __ call_Unimplemented();
 3740     }
 3741   %}
 3742 
 3743   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3744     C2_MacroAssembler _masm(&cbuf);
 3745 
 3746     // some calls to generated routines (arraycopy code) are scheduled
 3747     // by C2 as runtime calls. if so we can call them using a br (they
 3748     // will be in a reachable segment) otherwise we have to use a blr
 3749     // which loads the absolute address into a register.
 3750     address entry = (address)$meth$$method;
 3751     CodeBlob *cb = CodeCache::find_blob(entry);
 3752     if (cb) {
 3753       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3754       if (call == NULL) {
 3755         ciEnv::current()->record_failure("CodeCache is full");
 3756         return;
 3757       }
 3758     } else {
 3759       Label retaddr;
 3760       __ adr(rscratch2, retaddr);
 3761       __ lea(rscratch1, RuntimeAddress(entry));
 3762       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3763       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3764       __ blr(rscratch1);
 3765       __ bind(retaddr);
 3766       __ add(sp, sp, 2 * wordSize);
 3767     }
 3768     if (Compile::current()->max_vector_size() >= 16) {
 3769       __ reinitialize_ptrue();
 3770     }
 3771   %}
 3772 
 3773   enc_class aarch64_enc_rethrow() %{
 3774     C2_MacroAssembler _masm(&cbuf);
 3775     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3776   %}
 3777 
 3778   enc_class aarch64_enc_ret() %{
 3779     C2_MacroAssembler _masm(&cbuf);
 3780 #ifdef ASSERT
 3781     if (Compile::current()->max_vector_size() >= 16) {
 3782       __ verify_ptrue();
 3783     }
 3784 #endif
 3785     __ ret(lr);
 3786   %}
 3787 
 3788   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3789     C2_MacroAssembler _masm(&cbuf);
 3790     Register target_reg = as_Register($jump_target$$reg);
 3791     __ br(target_reg);
 3792   %}
 3793 
 3794   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3795     C2_MacroAssembler _masm(&cbuf);
 3796     Register target_reg = as_Register($jump_target$$reg);
 3797     // exception oop should be in r0
 3798     // ret addr has been popped into lr
 3799     // callee expects it in r3
 3800     __ mov(r3, lr);
 3801     __ br(target_reg);
 3802   %}
 3803 
 3804   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3805     C2_MacroAssembler _masm(&cbuf);
 3806     Register oop = as_Register($object$$reg);
 3807     Register box = as_Register($box$$reg);
 3808     Register disp_hdr = as_Register($tmp$$reg);
 3809     Register tmp = as_Register($tmp2$$reg);
 3810     Label cont;
 3811     Label object_has_monitor;
 3812     Label cas_failed;
 3813 
 3814     assert_different_registers(oop, box, tmp, disp_hdr);
 3815 
 3816     // Load markWord from object into displaced_header.
 3817     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3818 
 3819     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3820       __ load_klass(tmp, oop);
 3821       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3822       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3823       __ br(Assembler::NE, cont);
 3824     }
 3825 
 3826     if (UseBiasedLocking && !UseOptoBiasInlining) {
 3827       __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
 3828     }
 3829 
 3830     // Check for existing monitor
 3831     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3832 
 3833     if (LockingMode == LM_MONITOR) {
 3834       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3835       __ b(cont);
 3836     } else if (LockingMode == LM_LEGACY) {
 3837       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3838       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3839 
 3840       // Initialize the box. (Must happen before we update the object mark!)
 3841       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3842 
 3843       // Compare object markWord with an unlocked value (tmp) and if
 3844       // equal exchange the stack address of our box with object markWord.
 3845       // On failure disp_hdr contains the possibly locked markWord.
 3846       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3847                  /*release*/ true, /*weak*/ false, disp_hdr);
 3848       __ br(Assembler::EQ, cont);
 3849 
 3850       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3851 
 3852       // If the compare-and-exchange succeeded, then we found an unlocked
 3853       // object, will have now locked it will continue at label cont
 3854 
 3855       __ bind(cas_failed);
 3856       // We did not see an unlocked object so try the fast recursive case.
 3857 
 3858       // Check if the owner is self by comparing the value in the
 3859       // markWord of object (disp_hdr) with the stack pointer.
 3860       __ mov(rscratch1, sp);
 3861       __ sub(disp_hdr, disp_hdr, rscratch1);
 3862       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3863       // If condition is true we are cont and hence we can store 0 as the
 3864       // displaced header in the box, which indicates that it is a recursive lock.
 3865       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3866       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3867       __ b(cont);
 3868     } else {
 3869       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 3870       __ fast_lock(oop, disp_hdr, tmp, rscratch1, cont);
 3871       __ b(cont);
 3872     }
 3873 
 3874     // Handle existing monitor.
 3875     __ bind(object_has_monitor);
 3876 
 3877     // The object's monitor m is unlocked iff m->owner == NULL,
 3878     // otherwise m->owner may contain a thread or a stack address.
 3879     //
 3880     // Try to CAS m->owner from NULL to current thread.
 3881     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3882     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3883                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3884 
 3885     if (LockingMode != LM_LIGHTWEIGHT) {
 3886       // Store a non-null value into the box to avoid looking like a re-entrant
 3887       // lock. The fast-path monitor unlock code checks for
 3888       // markWord::monitor_value so use markWord::unused_mark which has the
 3889       // relevant bit set, and also matches ObjectSynchronizer::enter.
 3890       __ mov(tmp, (address)markWord::unused_mark().value());
 3891       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3892     }
 3893     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3894 
 3895     __ cmp(rscratch1, rthread);
 3896     __ br(Assembler::NE, cont); // Check for recursive locking
 3897 
 3898     // Recursive lock case
 3899     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3900     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3901 
 3902     __ bind(cont);
 3903     // flag == EQ indicates success
 3904     // flag == NE indicates failure
 3905   %}
 3906 
 3907   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3908     C2_MacroAssembler _masm(&cbuf);
 3909     Register oop = as_Register($object$$reg);
 3910     Register box = as_Register($box$$reg);
 3911     Register disp_hdr = as_Register($tmp$$reg);
 3912     Register tmp = as_Register($tmp2$$reg);
 3913     Label cont;
 3914     Label object_has_monitor;
 3915 
 3916     assert_different_registers(oop, box, tmp, disp_hdr);
 3917 
 3918     if (UseBiasedLocking && !UseOptoBiasInlining) {
 3919       __ biased_locking_exit(oop, tmp, cont);
 3920     }
 3921 
 3922     if (LockingMode == LM_LEGACY) {
 3923       // Find the lock address and load the displaced header from the stack.
 3924       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3925 
 3926       // If the displaced header is 0, we have a recursive unlock.
 3927       __ cmp(disp_hdr, zr);
 3928       __ br(Assembler::EQ, cont);
 3929     }
 3930 
 3931     // Handle existing monitor.
 3932     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3933     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3934 
 3935     if (LockingMode == LM_MONITOR) {
 3936       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3937       __ b(cont);
 3938     } else if (LockingMode == LM_LEGACY) {
 3939       // Check if it is still a light weight lock, this is is true if we
 3940       // see the stack address of the basicLock in the markWord of the
 3941       // object.
 3942 
 3943       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3944                  /*release*/ true, /*weak*/ false, tmp);
 3945       __ b(cont);
 3946     } else {
 3947       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 3948       __ fast_unlock(oop, tmp, box, disp_hdr, cont);
 3949       __ b(cont);
 3950     }
 3951 
 3952     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3953 
 3954     // Handle existing monitor.
 3955     __ bind(object_has_monitor);
 3956     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3957     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3958 
 3959     if (LockingMode == LM_LIGHTWEIGHT) {
 3960       // If the owner is anonymous, we need to fix it -- in an outline stub.
 3961       Register tmp2 = disp_hdr;
 3962       __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3963       // We cannot use tbnz here, the target might be too far away and cannot
 3964       // be encoded.
 3965       __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
 3966       C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
 3967       Compile::current()->output()->add_stub(stub);
 3968       __ br(Assembler::NE, stub->entry());
 3969       __ bind(stub->continuation());
 3970     }
 3971 
 3972     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3973 
 3974     Label notRecursive;
 3975     __ cbz(disp_hdr, notRecursive);
 3976 
 3977     // Recursive lock
 3978     __ sub(disp_hdr, disp_hdr, 1u);
 3979     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3980     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3981     __ b(cont);
 3982 
 3983     __ bind(notRecursive);
 3984     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3985     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3986     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3987     __ cmp(rscratch1, zr); // Sets flags for result
 3988     __ cbnz(rscratch1, cont);
 3989     // need a release store here
 3990     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3991     __ stlr(zr, tmp); // set unowned
 3992 
 3993     __ bind(cont);
 3994     // flag == EQ indicates success
 3995     // flag == NE indicates failure
 3996   %}
 3997 
 3998 %}
 3999 
 4000 //----------FRAME--------------------------------------------------------------
 4001 // Definition of frame structure and management information.
 4002 //
 4003 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4004 //                             |   (to get allocators register number
 4005 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4006 //  r   CALLER     |        |
 4007 //  o     |        +--------+      pad to even-align allocators stack-slot
 4008 //  w     V        |  pad0  |        numbers; owned by CALLER
 4009 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4010 //  h     ^        |   in   |  5
 4011 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4012 //  |     |        |        |  3
 4013 //  |     |        +--------+
 4014 //  V     |        | old out|      Empty on Intel, window on Sparc
 4015 //        |    old |preserve|      Must be even aligned.
 4016 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4017 //        |        |   in   |  3   area for Intel ret address
 4018 //     Owned by    |preserve|      Empty on Sparc.
 4019 //       SELF      +--------+
 4020 //        |        |  pad2  |  2   pad to align old SP
 4021 //        |        +--------+  1
 4022 //        |        | locks  |  0
 4023 //        |        +--------+----> OptoReg::stack0(), even aligned
 4024 //        |        |  pad1  | 11   pad to align new SP
 4025 //        |        +--------+
 4026 //        |        |        | 10
 4027 //        |        | spills |  9   spills
 4028 //        V        |        |  8   (pad0 slot for callee)
 4029 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4030 //        ^        |  out   |  7
 4031 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4032 //     Owned by    +--------+
 4033 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4034 //        |    new |preserve|      Must be even-aligned.
 4035 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4036 //        |        |        |
 4037 //
 4038 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4039 //         known from SELF's arguments and the Java calling convention.
 4040 //         Region 6-7 is determined per call site.
 4041 // Note 2: If the calling convention leaves holes in the incoming argument
 4042 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4043 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4044 //         incoming area, as the Java calling convention is completely under
 4045 //         the control of the AD file.  Doubles can be sorted and packed to
 4046 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4047 //         varargs C calling conventions.
 4048 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4049 //         even aligned with pad0 as needed.
 4050 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4051 //           (the latter is true on Intel but is it false on AArch64?)
 4052 //         region 6-11 is even aligned; it may be padded out more so that
 4053 //         the region from SP to FP meets the minimum stack alignment.
 4054 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4055 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4056 //         SP meets the minimum alignment.
 4057 
 4058 frame %{
 4059   // These three registers define part of the calling convention
 4060   // between compiled code and the interpreter.
 4061 
 4062   // Inline Cache Register or Method for I2C.
 4063   inline_cache_reg(R12);
 4064 
 4065   // Number of stack slots consumed by locking an object
 4066   sync_stack_slots(2);
 4067 
 4068   // Compiled code's Frame Pointer
 4069   frame_pointer(R31);
 4070 
 4071   // Interpreter stores its frame pointer in a register which is
 4072   // stored to the stack by I2CAdaptors.
 4073   // I2CAdaptors convert from interpreted java to compiled java.
 4074   interpreter_frame_pointer(R29);
 4075 
 4076   // Stack alignment requirement
 4077   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4078 
 4079   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4080   // for calls to C.  Supports the var-args backing area for register parms.
 4081   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4082 
 4083   // The after-PROLOG location of the return address.  Location of
 4084   // return address specifies a type (REG or STACK) and a number
 4085   // representing the register number (i.e. - use a register name) or
 4086   // stack slot.
 4087   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4088   // Otherwise, it is above the locks and verification slot and alignment word
 4089   // TODO this may well be correct but need to check why that - 2 is there
 4090   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4091   // which folds in the space used for monitors
 4092   return_addr(STACK - 2 +
 4093               align_up((Compile::current()->in_preserve_stack_slots() +
 4094                         Compile::current()->fixed_slots()),
 4095                        stack_alignment_in_slots()));
 4096 
 4097   // Location of compiled Java return values.  Same as C for now.
 4098   return_value
 4099   %{
 4100     // TODO do we allow ideal_reg == Op_RegN???
 4101     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4102            "only return normal values");
 4103 
 4104     static const int lo[Op_RegL + 1] = { // enum name
 4105       0,                                 // Op_Node
 4106       0,                                 // Op_Set
 4107       R0_num,                            // Op_RegN
 4108       R0_num,                            // Op_RegI
 4109       R0_num,                            // Op_RegP
 4110       V0_num,                            // Op_RegF
 4111       V0_num,                            // Op_RegD
 4112       R0_num                             // Op_RegL
 4113     };
 4114 
 4115     static const int hi[Op_RegL + 1] = { // enum name
 4116       0,                                 // Op_Node
 4117       0,                                 // Op_Set
 4118       OptoReg::Bad,                      // Op_RegN
 4119       OptoReg::Bad,                      // Op_RegI
 4120       R0_H_num,                          // Op_RegP
 4121       OptoReg::Bad,                      // Op_RegF
 4122       V0_H_num,                          // Op_RegD
 4123       R0_H_num                           // Op_RegL
 4124     };
 4125 
 4126     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4127   %}
 4128 %}
 4129 
 4130 //----------ATTRIBUTES---------------------------------------------------------
 4131 //----------Operand Attributes-------------------------------------------------
 4132 op_attrib op_cost(1);        // Required cost attribute
 4133 
 4134 //----------Instruction Attributes---------------------------------------------
 4135 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4136 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4137 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4138                                 // a non-matching short branch variant
 4139                                 // of some long branch?
 4140 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4141                                 // be a power of 2) specifies the
 4142                                 // alignment that some part of the
 4143                                 // instruction (not necessarily the
 4144                                 // start) requires.  If > 1, a
 4145                                 // compute_padding() function must be
 4146                                 // provided for the instruction
 4147 
 4148 //----------OPERANDS-----------------------------------------------------------
 4149 // Operand definitions must precede instruction definitions for correct parsing
 4150 // in the ADLC because operands constitute user defined types which are used in
 4151 // instruction definitions.
 4152 
 4153 //----------Simple Operands----------------------------------------------------
 4154 
 4155 // Integer operands 32 bit
 4156 // 32 bit immediate
 4157 operand immI()
 4158 %{
 4159   match(ConI);
 4160 
 4161   op_cost(0);
 4162   format %{ %}
 4163   interface(CONST_INTER);
 4164 %}
 4165 
 4166 // 32 bit zero
 4167 operand immI0()
 4168 %{
 4169   predicate(n->get_int() == 0);
 4170   match(ConI);
 4171 
 4172   op_cost(0);
 4173   format %{ %}
 4174   interface(CONST_INTER);
 4175 %}
 4176 
 4177 // 32 bit unit increment
 4178 operand immI_1()
 4179 %{
 4180   predicate(n->get_int() == 1);
 4181   match(ConI);
 4182 
 4183   op_cost(0);
 4184   format %{ %}
 4185   interface(CONST_INTER);
 4186 %}
 4187 
 4188 // 32 bit unit decrement
 4189 operand immI_M1()
 4190 %{
 4191   predicate(n->get_int() == -1);
 4192   match(ConI);
 4193 
 4194   op_cost(0);
 4195   format %{ %}
 4196   interface(CONST_INTER);
 4197 %}
 4198 
 4199 // Shift values for add/sub extension shift
 4200 operand immIExt()
 4201 %{
 4202   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4203   match(ConI);
 4204 
 4205   op_cost(0);
 4206   format %{ %}
 4207   interface(CONST_INTER);
 4208 %}
 4209 
 4210 operand immI_le_4()
 4211 %{
 4212   predicate(n->get_int() <= 4);
 4213   match(ConI);
 4214 
 4215   op_cost(0);
 4216   format %{ %}
 4217   interface(CONST_INTER);
 4218 %}
 4219 
 4220 operand immI_31()
 4221 %{
 4222   predicate(n->get_int() == 31);
 4223   match(ConI);
 4224 
 4225   op_cost(0);
 4226   format %{ %}
 4227   interface(CONST_INTER);
 4228 %}
 4229 
 4230 operand immI_2()
 4231 %{
 4232   predicate(n->get_int() == 2);
 4233   match(ConI);
 4234 
 4235   op_cost(0);
 4236   format %{ %}
 4237   interface(CONST_INTER);
 4238 %}
 4239 
 4240 operand immI_4()
 4241 %{
 4242   predicate(n->get_int() == 4);
 4243   match(ConI);
 4244 
 4245   op_cost(0);
 4246   format %{ %}
 4247   interface(CONST_INTER);
 4248 %}
 4249 
 4250 operand immI_8()
 4251 %{
 4252   predicate(n->get_int() == 8);
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 operand immI_16()
 4261 %{
 4262   predicate(n->get_int() == 16);
 4263   match(ConI);
 4264 
 4265   op_cost(0);
 4266   format %{ %}
 4267   interface(CONST_INTER);
 4268 %}
 4269 
 4270 operand immI_24()
 4271 %{
 4272   predicate(n->get_int() == 24);
 4273   match(ConI);
 4274 
 4275   op_cost(0);
 4276   format %{ %}
 4277   interface(CONST_INTER);
 4278 %}
 4279 
 4280 operand immI_32()
 4281 %{
 4282   predicate(n->get_int() == 32);
 4283   match(ConI);
 4284 
 4285   op_cost(0);
 4286   format %{ %}
 4287   interface(CONST_INTER);
 4288 %}
 4289 
 4290 operand immI_48()
 4291 %{
 4292   predicate(n->get_int() == 48);
 4293   match(ConI);
 4294 
 4295   op_cost(0);
 4296   format %{ %}
 4297   interface(CONST_INTER);
 4298 %}
 4299 
 4300 operand immI_56()
 4301 %{
 4302   predicate(n->get_int() == 56);
 4303   match(ConI);
 4304 
 4305   op_cost(0);
 4306   format %{ %}
 4307   interface(CONST_INTER);
 4308 %}
 4309 
 4310 operand immI_63()
 4311 %{
 4312   predicate(n->get_int() == 63);
 4313   match(ConI);
 4314 
 4315   op_cost(0);
 4316   format %{ %}
 4317   interface(CONST_INTER);
 4318 %}
 4319 
 4320 operand immI_64()
 4321 %{
 4322   predicate(n->get_int() == 64);
 4323   match(ConI);
 4324 
 4325   op_cost(0);
 4326   format %{ %}
 4327   interface(CONST_INTER);
 4328 %}
 4329 
 4330 operand immI_255()
 4331 %{
 4332   predicate(n->get_int() == 255);
 4333   match(ConI);
 4334 
 4335   op_cost(0);
 4336   format %{ %}
 4337   interface(CONST_INTER);
 4338 %}
 4339 
 4340 operand immI_65535()
 4341 %{
 4342   predicate(n->get_int() == 65535);
 4343   match(ConI);
 4344 
 4345   op_cost(0);
 4346   format %{ %}
 4347   interface(CONST_INTER);
 4348 %}
 4349 
 4350 operand immI_positive()
 4351 %{
 4352   predicate(n->get_int() > 0);
 4353   match(ConI);
 4354 
 4355   op_cost(0);
 4356   format %{ %}
 4357   interface(CONST_INTER);
 4358 %}
 4359 
 4360 operand immL_255()
 4361 %{
 4362   predicate(n->get_long() == 255L);
 4363   match(ConL);
 4364 
 4365   op_cost(0);
 4366   format %{ %}
 4367   interface(CONST_INTER);
 4368 %}
 4369 
 4370 operand immL_65535()
 4371 %{
 4372   predicate(n->get_long() == 65535L);
 4373   match(ConL);
 4374 
 4375   op_cost(0);
 4376   format %{ %}
 4377   interface(CONST_INTER);
 4378 %}
 4379 
 4380 operand immL_4294967295()
 4381 %{
 4382   predicate(n->get_long() == 4294967295L);
 4383   match(ConL);
 4384 
 4385   op_cost(0);
 4386   format %{ %}
 4387   interface(CONST_INTER);
 4388 %}
 4389 
 4390 operand immL_bitmask()
 4391 %{
 4392   predicate((n->get_long() != 0)
 4393             && ((n->get_long() & 0xc000000000000000l) == 0)
 4394             && is_power_of_2(n->get_long() + 1));
 4395   match(ConL);
 4396 
 4397   op_cost(0);
 4398   format %{ %}
 4399   interface(CONST_INTER);
 4400 %}
 4401 
 4402 operand immI_bitmask()
 4403 %{
 4404   predicate((n->get_int() != 0)
 4405             && ((n->get_int() & 0xc0000000) == 0)
 4406             && is_power_of_2(n->get_int() + 1));
 4407   match(ConI);
 4408 
 4409   op_cost(0);
 4410   format %{ %}
 4411   interface(CONST_INTER);
 4412 %}
 4413 
 4414 operand immL_positive_bitmaskI()
 4415 %{
 4416   predicate((n->get_long() != 0)
 4417             && ((julong)n->get_long() < 0x80000000ULL)
 4418             && is_power_of_2(n->get_long() + 1));
 4419   match(ConL);
 4420 
 4421   op_cost(0);
 4422   format %{ %}
 4423   interface(CONST_INTER);
 4424 %}
 4425 
 4426 // Scale values for scaled offset addressing modes (up to long but not quad)
 4427 operand immIScale()
 4428 %{
 4429   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4430   match(ConI);
 4431 
 4432   op_cost(0);
 4433   format %{ %}
 4434   interface(CONST_INTER);
 4435 %}
 4436 
 4437 // 26 bit signed offset -- for pc-relative branches
 4438 operand immI26()
 4439 %{
 4440   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4441   match(ConI);
 4442 
 4443   op_cost(0);
 4444   format %{ %}
 4445   interface(CONST_INTER);
 4446 %}
 4447 
 4448 // 19 bit signed offset -- for pc-relative loads
 4449 operand immI19()
 4450 %{
 4451   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4452   match(ConI);
 4453 
 4454   op_cost(0);
 4455   format %{ %}
 4456   interface(CONST_INTER);
 4457 %}
 4458 
 4459 // 12 bit unsigned offset -- for base plus immediate loads
 4460 operand immIU12()
 4461 %{
 4462   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4463   match(ConI);
 4464 
 4465   op_cost(0);
 4466   format %{ %}
 4467   interface(CONST_INTER);
 4468 %}
 4469 
 4470 operand immLU12()
 4471 %{
 4472   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4473   match(ConL);
 4474 
 4475   op_cost(0);
 4476   format %{ %}
 4477   interface(CONST_INTER);
 4478 %}
 4479 
 4480 // Offset for scaled or unscaled immediate loads and stores
 4481 operand immIOffset()
 4482 %{
 4483   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4484   match(ConI);
 4485 
 4486   op_cost(0);
 4487   format %{ %}
 4488   interface(CONST_INTER);
 4489 %}
 4490 
 4491 operand immIOffset1()
 4492 %{
 4493   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4494   match(ConI);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 operand immIOffset2()
 4502 %{
 4503   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4504   match(ConI);
 4505 
 4506   op_cost(0);
 4507   format %{ %}
 4508   interface(CONST_INTER);
 4509 %}
 4510 
 4511 operand immIOffset4()
 4512 %{
 4513   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4514   match(ConI);
 4515 
 4516   op_cost(0);
 4517   format %{ %}
 4518   interface(CONST_INTER);
 4519 %}
 4520 
 4521 operand immIOffset8()
 4522 %{
 4523   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4524   match(ConI);
 4525 
 4526   op_cost(0);
 4527   format %{ %}
 4528   interface(CONST_INTER);
 4529 %}
 4530 
 4531 operand immIOffset16()
 4532 %{
 4533   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4534   match(ConI);
 4535 
 4536   op_cost(0);
 4537   format %{ %}
 4538   interface(CONST_INTER);
 4539 %}
 4540 
 4541 operand immLoffset()
 4542 %{
 4543   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4544   match(ConL);
 4545 
 4546   op_cost(0);
 4547   format %{ %}
 4548   interface(CONST_INTER);
 4549 %}
 4550 
 4551 operand immLoffset1()
 4552 %{
 4553   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4554   match(ConL);
 4555 
 4556   op_cost(0);
 4557   format %{ %}
 4558   interface(CONST_INTER);
 4559 %}
 4560 
 4561 operand immLoffset2()
 4562 %{
 4563   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4564   match(ConL);
 4565 
 4566   op_cost(0);
 4567   format %{ %}
 4568   interface(CONST_INTER);
 4569 %}
 4570 
 4571 operand immLoffset4()
 4572 %{
 4573   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4574   match(ConL);
 4575 
 4576   op_cost(0);
 4577   format %{ %}
 4578   interface(CONST_INTER);
 4579 %}
 4580 
 4581 operand immLoffset8()
 4582 %{
 4583   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4584   match(ConL);
 4585 
 4586   op_cost(0);
 4587   format %{ %}
 4588   interface(CONST_INTER);
 4589 %}
 4590 
 4591 operand immLoffset16()
 4592 %{
 4593   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4594   match(ConL);
 4595 
 4596   op_cost(0);
 4597   format %{ %}
 4598   interface(CONST_INTER);
 4599 %}
 4600 
 4601 // 8 bit signed value.
 4602 operand immI8()
 4603 %{
 4604   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4605   match(ConI);
 4606 
 4607   op_cost(0);
 4608   format %{ %}
 4609   interface(CONST_INTER);
 4610 %}
 4611 
 4612 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4613 operand immI8_shift8()
 4614 %{
 4615   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4616             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4617   match(ConI);
 4618 
 4619   op_cost(0);
 4620   format %{ %}
 4621   interface(CONST_INTER);
 4622 %}
 4623 
 4624 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4625 operand immL8_shift8()
 4626 %{
 4627   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4628             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4629   match(ConL);
 4630 
 4631   op_cost(0);
 4632   format %{ %}
 4633   interface(CONST_INTER);
 4634 %}
 4635 
 4636 // 32 bit integer valid for add sub immediate
 4637 operand immIAddSub()
 4638 %{
 4639   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4640   match(ConI);
 4641   op_cost(0);
 4642   format %{ %}
 4643   interface(CONST_INTER);
 4644 %}
 4645 
 4646 // 32 bit unsigned integer valid for logical immediate
 4647 // TODO -- check this is right when e.g the mask is 0x80000000
 4648 operand immILog()
 4649 %{
 4650   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4651   match(ConI);
 4652 
 4653   op_cost(0);
 4654   format %{ %}
 4655   interface(CONST_INTER);
 4656 %}
 4657 
 4658 // Integer operands 64 bit
 4659 // 64 bit immediate
 4660 operand immL()
 4661 %{
 4662   match(ConL);
 4663 
 4664   op_cost(0);
 4665   format %{ %}
 4666   interface(CONST_INTER);
 4667 %}
 4668 
 4669 // 64 bit zero
 4670 operand immL0()
 4671 %{
 4672   predicate(n->get_long() == 0);
 4673   match(ConL);
 4674 
 4675   op_cost(0);
 4676   format %{ %}
 4677   interface(CONST_INTER);
 4678 %}
 4679 
 4680 // 64 bit unit increment
 4681 operand immL_1()
 4682 %{
 4683   predicate(n->get_long() == 1);
 4684   match(ConL);
 4685 
 4686   op_cost(0);
 4687   format %{ %}
 4688   interface(CONST_INTER);
 4689 %}
 4690 
 4691 // 64 bit unit decrement
 4692 operand immL_M1()
 4693 %{
 4694   predicate(n->get_long() == -1);
 4695   match(ConL);
 4696 
 4697   op_cost(0);
 4698   format %{ %}
 4699   interface(CONST_INTER);
 4700 %}
 4701 
 4702 // 32 bit offset of pc in thread anchor
 4703 
 4704 operand immL_pc_off()
 4705 %{
 4706   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4707                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4708   match(ConL);
 4709 
 4710   op_cost(0);
 4711   format %{ %}
 4712   interface(CONST_INTER);
 4713 %}
 4714 
 4715 // 64 bit integer valid for add sub immediate
 4716 operand immLAddSub()
 4717 %{
 4718   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4719   match(ConL);
 4720   op_cost(0);
 4721   format %{ %}
 4722   interface(CONST_INTER);
 4723 %}
 4724 
 4725 // 64 bit integer valid for logical immediate
 4726 operand immLLog()
 4727 %{
 4728   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4729   match(ConL);
 4730   op_cost(0);
 4731   format %{ %}
 4732   interface(CONST_INTER);
 4733 %}
 4734 
 4735 // Long Immediate: low 32-bit mask
 4736 operand immL_32bits()
 4737 %{
 4738   predicate(n->get_long() == 0xFFFFFFFFL);
 4739   match(ConL);
 4740   op_cost(0);
 4741   format %{ %}
 4742   interface(CONST_INTER);
 4743 %}
 4744 
 4745 // Pointer operands
 4746 // Pointer Immediate
 4747 operand immP()
 4748 %{
 4749   match(ConP);
 4750 
 4751   op_cost(0);
 4752   format %{ %}
 4753   interface(CONST_INTER);
 4754 %}
 4755 
 4756 // NULL Pointer Immediate
 4757 operand immP0()
 4758 %{
 4759   predicate(n->get_ptr() == 0);
 4760   match(ConP);
 4761 
 4762   op_cost(0);
 4763   format %{ %}
 4764   interface(CONST_INTER);
 4765 %}
 4766 
 4767 // Pointer Immediate One
 4768 // this is used in object initialization (initial object header)
 4769 operand immP_1()
 4770 %{
 4771   predicate(n->get_ptr() == 1);
 4772   match(ConP);
 4773 
 4774   op_cost(0);
 4775   format %{ %}
 4776   interface(CONST_INTER);
 4777 %}
 4778 
 4779 // Card Table Byte Map Base
 4780 operand immByteMapBase()
 4781 %{
 4782   // Get base of card map
 4783   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4784             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4785   match(ConP);
 4786 
 4787   op_cost(0);
 4788   format %{ %}
 4789   interface(CONST_INTER);
 4790 %}
 4791 
 4792 // Pointer Immediate Minus One
 4793 // this is used when we want to write the current PC to the thread anchor
 4794 operand immP_M1()
 4795 %{
 4796   predicate(n->get_ptr() == -1);
 4797   match(ConP);
 4798 
 4799   op_cost(0);
 4800   format %{ %}
 4801   interface(CONST_INTER);
 4802 %}
 4803 
 4804 // Pointer Immediate Minus Two
 4805 // this is used when we want to write the current PC to the thread anchor
 4806 operand immP_M2()
 4807 %{
 4808   predicate(n->get_ptr() == -2);
 4809   match(ConP);
 4810 
 4811   op_cost(0);
 4812   format %{ %}
 4813   interface(CONST_INTER);
 4814 %}
 4815 
 4816 // Float and Double operands
 4817 // Double Immediate
 4818 operand immD()
 4819 %{
 4820   match(ConD);
 4821   op_cost(0);
 4822   format %{ %}
 4823   interface(CONST_INTER);
 4824 %}
 4825 
 4826 // Double Immediate: +0.0d
 4827 operand immD0()
 4828 %{
 4829   predicate(jlong_cast(n->getd()) == 0);
 4830   match(ConD);
 4831 
 4832   op_cost(0);
 4833   format %{ %}
 4834   interface(CONST_INTER);
 4835 %}
 4836 
 4837 // constant 'double +0.0'.
 4838 operand immDPacked()
 4839 %{
 4840   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4841   match(ConD);
 4842   op_cost(0);
 4843   format %{ %}
 4844   interface(CONST_INTER);
 4845 %}
 4846 
 4847 // Float Immediate
 4848 operand immF()
 4849 %{
 4850   match(ConF);
 4851   op_cost(0);
 4852   format %{ %}
 4853   interface(CONST_INTER);
 4854 %}
 4855 
 4856 // Float Immediate: +0.0f.
 4857 operand immF0()
 4858 %{
 4859   predicate(jint_cast(n->getf()) == 0);
 4860   match(ConF);
 4861 
 4862   op_cost(0);
 4863   format %{ %}
 4864   interface(CONST_INTER);
 4865 %}
 4866 
 4867 //
 4868 operand immFPacked()
 4869 %{
 4870   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4871   match(ConF);
 4872   op_cost(0);
 4873   format %{ %}
 4874   interface(CONST_INTER);
 4875 %}
 4876 
 4877 // Narrow pointer operands
 4878 // Narrow Pointer Immediate
 4879 operand immN()
 4880 %{
 4881   match(ConN);
 4882 
 4883   op_cost(0);
 4884   format %{ %}
 4885   interface(CONST_INTER);
 4886 %}
 4887 
 4888 // Narrow NULL Pointer Immediate
 4889 operand immN0()
 4890 %{
 4891   predicate(n->get_narrowcon() == 0);
 4892   match(ConN);
 4893 
 4894   op_cost(0);
 4895   format %{ %}
 4896   interface(CONST_INTER);
 4897 %}
 4898 
 4899 operand immNKlass()
 4900 %{
 4901   match(ConNKlass);
 4902 
 4903   op_cost(0);
 4904   format %{ %}
 4905   interface(CONST_INTER);
 4906 %}
 4907 
 4908 // Integer 32 bit Register Operands
 4909 // Integer 32 bitRegister (excludes SP)
 4910 operand iRegI()
 4911 %{
 4912   constraint(ALLOC_IN_RC(any_reg32));
 4913   match(RegI);
 4914   match(iRegINoSp);
 4915   op_cost(0);
 4916   format %{ %}
 4917   interface(REG_INTER);
 4918 %}
 4919 
 4920 // Integer 32 bit Register not Special
 4921 operand iRegINoSp()
 4922 %{
 4923   constraint(ALLOC_IN_RC(no_special_reg32));
 4924   match(RegI);
 4925   op_cost(0);
 4926   format %{ %}
 4927   interface(REG_INTER);
 4928 %}
 4929 
 4930 // Integer 64 bit Register Operands
 4931 // Integer 64 bit Register (includes SP)
 4932 operand iRegL()
 4933 %{
 4934   constraint(ALLOC_IN_RC(any_reg));
 4935   match(RegL);
 4936   match(iRegLNoSp);
 4937   op_cost(0);
 4938   format %{ %}
 4939   interface(REG_INTER);
 4940 %}
 4941 
 4942 // Integer 64 bit Register not Special
 4943 operand iRegLNoSp()
 4944 %{
 4945   constraint(ALLOC_IN_RC(no_special_reg));
 4946   match(RegL);
 4947   match(iRegL_R0);
 4948   format %{ %}
 4949   interface(REG_INTER);
 4950 %}
 4951 
 4952 // Pointer Register Operands
 4953 // Pointer Register
 4954 operand iRegP()
 4955 %{
 4956   constraint(ALLOC_IN_RC(ptr_reg));
 4957   match(RegP);
 4958   match(iRegPNoSp);
 4959   match(iRegP_R0);
 4960   //match(iRegP_R2);
 4961   //match(iRegP_R4);
 4962   match(iRegP_R5);
 4963   match(thread_RegP);
 4964   op_cost(0);
 4965   format %{ %}
 4966   interface(REG_INTER);
 4967 %}
 4968 
 4969 // Pointer 64 bit Register not Special
 4970 operand iRegPNoSp()
 4971 %{
 4972   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4973   match(RegP);
 4974   // match(iRegP);
 4975   // match(iRegP_R0);
 4976   // match(iRegP_R2);
 4977   // match(iRegP_R4);
 4978   // match(iRegP_R5);
 4979   // match(thread_RegP);
 4980   op_cost(0);
 4981   format %{ %}
 4982   interface(REG_INTER);
 4983 %}
 4984 
 4985 // Pointer 64 bit Register R0 only
 4986 operand iRegP_R0()
 4987 %{
 4988   constraint(ALLOC_IN_RC(r0_reg));
 4989   match(RegP);
 4990   // match(iRegP);
 4991   match(iRegPNoSp);
 4992   op_cost(0);
 4993   format %{ %}
 4994   interface(REG_INTER);
 4995 %}
 4996 
 4997 // Pointer 64 bit Register R1 only
 4998 operand iRegP_R1()
 4999 %{
 5000   constraint(ALLOC_IN_RC(r1_reg));
 5001   match(RegP);
 5002   // match(iRegP);
 5003   match(iRegPNoSp);
 5004   op_cost(0);
 5005   format %{ %}
 5006   interface(REG_INTER);
 5007 %}
 5008 
 5009 // Pointer 64 bit Register R2 only
 5010 operand iRegP_R2()
 5011 %{
 5012   constraint(ALLOC_IN_RC(r2_reg));
 5013   match(RegP);
 5014   // match(iRegP);
 5015   match(iRegPNoSp);
 5016   op_cost(0);
 5017   format %{ %}
 5018   interface(REG_INTER);
 5019 %}
 5020 
 5021 // Pointer 64 bit Register R3 only
 5022 operand iRegP_R3()
 5023 %{
 5024   constraint(ALLOC_IN_RC(r3_reg));
 5025   match(RegP);
 5026   // match(iRegP);
 5027   match(iRegPNoSp);
 5028   op_cost(0);
 5029   format %{ %}
 5030   interface(REG_INTER);
 5031 %}
 5032 
 5033 // Pointer 64 bit Register R4 only
 5034 operand iRegP_R4()
 5035 %{
 5036   constraint(ALLOC_IN_RC(r4_reg));
 5037   match(RegP);
 5038   // match(iRegP);
 5039   match(iRegPNoSp);
 5040   op_cost(0);
 5041   format %{ %}
 5042   interface(REG_INTER);
 5043 %}
 5044 
 5045 // Pointer 64 bit Register R5 only
 5046 operand iRegP_R5()
 5047 %{
 5048   constraint(ALLOC_IN_RC(r5_reg));
 5049   match(RegP);
 5050   // match(iRegP);
 5051   match(iRegPNoSp);
 5052   op_cost(0);
 5053   format %{ %}
 5054   interface(REG_INTER);
 5055 %}
 5056 
 5057 // Pointer 64 bit Register R10 only
 5058 operand iRegP_R10()
 5059 %{
 5060   constraint(ALLOC_IN_RC(r10_reg));
 5061   match(RegP);
 5062   // match(iRegP);
 5063   match(iRegPNoSp);
 5064   op_cost(0);
 5065   format %{ %}
 5066   interface(REG_INTER);
 5067 %}
 5068 
 5069 // Long 64 bit Register R0 only
 5070 operand iRegL_R0()
 5071 %{
 5072   constraint(ALLOC_IN_RC(r0_reg));
 5073   match(RegL);
 5074   match(iRegLNoSp);
 5075   op_cost(0);
 5076   format %{ %}
 5077   interface(REG_INTER);
 5078 %}
 5079 
 5080 // Long 64 bit Register R2 only
 5081 operand iRegL_R2()
 5082 %{
 5083   constraint(ALLOC_IN_RC(r2_reg));
 5084   match(RegL);
 5085   match(iRegLNoSp);
 5086   op_cost(0);
 5087   format %{ %}
 5088   interface(REG_INTER);
 5089 %}
 5090 
 5091 // Long 64 bit Register R3 only
 5092 operand iRegL_R3()
 5093 %{
 5094   constraint(ALLOC_IN_RC(r3_reg));
 5095   match(RegL);
 5096   match(iRegLNoSp);
 5097   op_cost(0);
 5098   format %{ %}
 5099   interface(REG_INTER);
 5100 %}
 5101 
 5102 // Long 64 bit Register R11 only
 5103 operand iRegL_R11()
 5104 %{
 5105   constraint(ALLOC_IN_RC(r11_reg));
 5106   match(RegL);
 5107   match(iRegLNoSp);
 5108   op_cost(0);
 5109   format %{ %}
 5110   interface(REG_INTER);
 5111 %}
 5112 
 5113 // Pointer 64 bit Register FP only
 5114 operand iRegP_FP()
 5115 %{
 5116   constraint(ALLOC_IN_RC(fp_reg));
 5117   match(RegP);
 5118   // match(iRegP);
 5119   op_cost(0);
 5120   format %{ %}
 5121   interface(REG_INTER);
 5122 %}
 5123 
 5124 // Register R0 only
 5125 operand iRegI_R0()
 5126 %{
 5127   constraint(ALLOC_IN_RC(int_r0_reg));
 5128   match(RegI);
 5129   match(iRegINoSp);
 5130   op_cost(0);
 5131   format %{ %}
 5132   interface(REG_INTER);
 5133 %}
 5134 
 5135 // Register R2 only
 5136 operand iRegI_R2()
 5137 %{
 5138   constraint(ALLOC_IN_RC(int_r2_reg));
 5139   match(RegI);
 5140   match(iRegINoSp);
 5141   op_cost(0);
 5142   format %{ %}
 5143   interface(REG_INTER);
 5144 %}
 5145 
 5146 // Register R3 only
 5147 operand iRegI_R3()
 5148 %{
 5149   constraint(ALLOC_IN_RC(int_r3_reg));
 5150   match(RegI);
 5151   match(iRegINoSp);
 5152   op_cost(0);
 5153   format %{ %}
 5154   interface(REG_INTER);
 5155 %}
 5156 
 5157 
 5158 // Register R4 only
 5159 operand iRegI_R4()
 5160 %{
 5161   constraint(ALLOC_IN_RC(int_r4_reg));
 5162   match(RegI);
 5163   match(iRegINoSp);
 5164   op_cost(0);
 5165   format %{ %}
 5166   interface(REG_INTER);
 5167 %}
 5168 
 5169 
 5170 // Pointer Register Operands
 5171 // Narrow Pointer Register
 5172 operand iRegN()
 5173 %{
 5174   constraint(ALLOC_IN_RC(any_reg32));
 5175   match(RegN);
 5176   match(iRegNNoSp);
 5177   op_cost(0);
 5178   format %{ %}
 5179   interface(REG_INTER);
 5180 %}
 5181 
 5182 operand iRegN_R0()
 5183 %{
 5184   constraint(ALLOC_IN_RC(r0_reg));
 5185   match(iRegN);
 5186   op_cost(0);
 5187   format %{ %}
 5188   interface(REG_INTER);
 5189 %}
 5190 
 5191 operand iRegN_R2()
 5192 %{
 5193   constraint(ALLOC_IN_RC(r2_reg));
 5194   match(iRegN);
 5195   op_cost(0);
 5196   format %{ %}
 5197   interface(REG_INTER);
 5198 %}
 5199 
 5200 operand iRegN_R3()
 5201 %{
 5202   constraint(ALLOC_IN_RC(r3_reg));
 5203   match(iRegN);
 5204   op_cost(0);
 5205   format %{ %}
 5206   interface(REG_INTER);
 5207 %}
 5208 
 5209 // Integer 64 bit Register not Special
 5210 operand iRegNNoSp()
 5211 %{
 5212   constraint(ALLOC_IN_RC(no_special_reg32));
 5213   match(RegN);
 5214   op_cost(0);
 5215   format %{ %}
 5216   interface(REG_INTER);
 5217 %}
 5218 
 5219 // heap base register -- used for encoding immN0
 5220 
 5221 operand iRegIHeapbase()
 5222 %{
 5223   constraint(ALLOC_IN_RC(heapbase_reg));
 5224   match(RegI);
 5225   op_cost(0);
 5226   format %{ %}
 5227   interface(REG_INTER);
 5228 %}
 5229 
 5230 // Float Register
 5231 // Float register operands
 5232 operand vRegF()
 5233 %{
 5234   constraint(ALLOC_IN_RC(float_reg));
 5235   match(RegF);
 5236 
 5237   op_cost(0);
 5238   format %{ %}
 5239   interface(REG_INTER);
 5240 %}
 5241 
 5242 // Double Register
 5243 // Double register operands
 5244 operand vRegD()
 5245 %{
 5246   constraint(ALLOC_IN_RC(double_reg));
 5247   match(RegD);
 5248 
 5249   op_cost(0);
 5250   format %{ %}
 5251   interface(REG_INTER);
 5252 %}
 5253 
 5254 // Generic vector class. This will be used for
 5255 // all vector operands, including NEON and SVE,
 5256 // but currently only used for SVE VecA.
 5257 operand vReg()
 5258 %{
 5259   constraint(ALLOC_IN_RC(vectora_reg));
 5260   match(VecA);
 5261   op_cost(0);
 5262   format %{ %}
 5263   interface(REG_INTER);
 5264 %}
 5265 
 5266 operand vecD()
 5267 %{
 5268   constraint(ALLOC_IN_RC(vectord_reg));
 5269   match(VecD);
 5270 
 5271   op_cost(0);
 5272   format %{ %}
 5273   interface(REG_INTER);
 5274 %}
 5275 
 5276 operand vecX()
 5277 %{
 5278   constraint(ALLOC_IN_RC(vectorx_reg));
 5279   match(VecX);
 5280 
 5281   op_cost(0);
 5282   format %{ %}
 5283   interface(REG_INTER);
 5284 %}
 5285 
 5286 operand vRegD_V0()
 5287 %{
 5288   constraint(ALLOC_IN_RC(v0_reg));
 5289   match(RegD);
 5290   op_cost(0);
 5291   format %{ %}
 5292   interface(REG_INTER);
 5293 %}
 5294 
 5295 operand vRegD_V1()
 5296 %{
 5297   constraint(ALLOC_IN_RC(v1_reg));
 5298   match(RegD);
 5299   op_cost(0);
 5300   format %{ %}
 5301   interface(REG_INTER);
 5302 %}
 5303 
 5304 operand vRegD_V2()
 5305 %{
 5306   constraint(ALLOC_IN_RC(v2_reg));
 5307   match(RegD);
 5308   op_cost(0);
 5309   format %{ %}
 5310   interface(REG_INTER);
 5311 %}
 5312 
 5313 operand vRegD_V3()
 5314 %{
 5315   constraint(ALLOC_IN_RC(v3_reg));
 5316   match(RegD);
 5317   op_cost(0);
 5318   format %{ %}
 5319   interface(REG_INTER);
 5320 %}
 5321 
 5322 operand vRegD_V4()
 5323 %{
 5324   constraint(ALLOC_IN_RC(v4_reg));
 5325   match(RegD);
 5326   op_cost(0);
 5327   format %{ %}
 5328   interface(REG_INTER);
 5329 %}
 5330 
 5331 operand vRegD_V5()
 5332 %{
 5333   constraint(ALLOC_IN_RC(v5_reg));
 5334   match(RegD);
 5335   op_cost(0);
 5336   format %{ %}
 5337   interface(REG_INTER);
 5338 %}
 5339 
 5340 operand vRegD_V6()
 5341 %{
 5342   constraint(ALLOC_IN_RC(v6_reg));
 5343   match(RegD);
 5344   op_cost(0);
 5345   format %{ %}
 5346   interface(REG_INTER);
 5347 %}
 5348 
 5349 operand vRegD_V7()
 5350 %{
 5351   constraint(ALLOC_IN_RC(v7_reg));
 5352   match(RegD);
 5353   op_cost(0);
 5354   format %{ %}
 5355   interface(REG_INTER);
 5356 %}
 5357 
 5358 operand vRegD_V8()
 5359 %{
 5360   constraint(ALLOC_IN_RC(v8_reg));
 5361   match(RegD);
 5362   op_cost(0);
 5363   format %{ %}
 5364   interface(REG_INTER);
 5365 %}
 5366 
 5367 operand vRegD_V9()
 5368 %{
 5369   constraint(ALLOC_IN_RC(v9_reg));
 5370   match(RegD);
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 operand vRegD_V10()
 5377 %{
 5378   constraint(ALLOC_IN_RC(v10_reg));
 5379   match(RegD);
 5380   op_cost(0);
 5381   format %{ %}
 5382   interface(REG_INTER);
 5383 %}
 5384 
 5385 operand vRegD_V11()
 5386 %{
 5387   constraint(ALLOC_IN_RC(v11_reg));
 5388   match(RegD);
 5389   op_cost(0);
 5390   format %{ %}
 5391   interface(REG_INTER);
 5392 %}
 5393 
 5394 operand vRegD_V12()
 5395 %{
 5396   constraint(ALLOC_IN_RC(v12_reg));
 5397   match(RegD);
 5398   op_cost(0);
 5399   format %{ %}
 5400   interface(REG_INTER);
 5401 %}
 5402 
 5403 operand vRegD_V13()
 5404 %{
 5405   constraint(ALLOC_IN_RC(v13_reg));
 5406   match(RegD);
 5407   op_cost(0);
 5408   format %{ %}
 5409   interface(REG_INTER);
 5410 %}
 5411 
 5412 operand vRegD_V14()
 5413 %{
 5414   constraint(ALLOC_IN_RC(v14_reg));
 5415   match(RegD);
 5416   op_cost(0);
 5417   format %{ %}
 5418   interface(REG_INTER);
 5419 %}
 5420 
 5421 operand vRegD_V15()
 5422 %{
 5423   constraint(ALLOC_IN_RC(v15_reg));
 5424   match(RegD);
 5425   op_cost(0);
 5426   format %{ %}
 5427   interface(REG_INTER);
 5428 %}
 5429 
 5430 operand vRegD_V16()
 5431 %{
 5432   constraint(ALLOC_IN_RC(v16_reg));
 5433   match(RegD);
 5434   op_cost(0);
 5435   format %{ %}
 5436   interface(REG_INTER);
 5437 %}
 5438 
 5439 operand vRegD_V17()
 5440 %{
 5441   constraint(ALLOC_IN_RC(v17_reg));
 5442   match(RegD);
 5443   op_cost(0);
 5444   format %{ %}
 5445   interface(REG_INTER);
 5446 %}
 5447 
 5448 operand vRegD_V18()
 5449 %{
 5450   constraint(ALLOC_IN_RC(v18_reg));
 5451   match(RegD);
 5452   op_cost(0);
 5453   format %{ %}
 5454   interface(REG_INTER);
 5455 %}
 5456 
 5457 operand vRegD_V19()
 5458 %{
 5459   constraint(ALLOC_IN_RC(v19_reg));
 5460   match(RegD);
 5461   op_cost(0);
 5462   format %{ %}
 5463   interface(REG_INTER);
 5464 %}
 5465 
 5466 operand vRegD_V20()
 5467 %{
 5468   constraint(ALLOC_IN_RC(v20_reg));
 5469   match(RegD);
 5470   op_cost(0);
 5471   format %{ %}
 5472   interface(REG_INTER);
 5473 %}
 5474 
 5475 operand vRegD_V21()
 5476 %{
 5477   constraint(ALLOC_IN_RC(v21_reg));
 5478   match(RegD);
 5479   op_cost(0);
 5480   format %{ %}
 5481   interface(REG_INTER);
 5482 %}
 5483 
 5484 operand vRegD_V22()
 5485 %{
 5486   constraint(ALLOC_IN_RC(v22_reg));
 5487   match(RegD);
 5488   op_cost(0);
 5489   format %{ %}
 5490   interface(REG_INTER);
 5491 %}
 5492 
 5493 operand vRegD_V23()
 5494 %{
 5495   constraint(ALLOC_IN_RC(v23_reg));
 5496   match(RegD);
 5497   op_cost(0);
 5498   format %{ %}
 5499   interface(REG_INTER);
 5500 %}
 5501 
 5502 operand vRegD_V24()
 5503 %{
 5504   constraint(ALLOC_IN_RC(v24_reg));
 5505   match(RegD);
 5506   op_cost(0);
 5507   format %{ %}
 5508   interface(REG_INTER);
 5509 %}
 5510 
 5511 operand vRegD_V25()
 5512 %{
 5513   constraint(ALLOC_IN_RC(v25_reg));
 5514   match(RegD);
 5515   op_cost(0);
 5516   format %{ %}
 5517   interface(REG_INTER);
 5518 %}
 5519 
 5520 operand vRegD_V26()
 5521 %{
 5522   constraint(ALLOC_IN_RC(v26_reg));
 5523   match(RegD);
 5524   op_cost(0);
 5525   format %{ %}
 5526   interface(REG_INTER);
 5527 %}
 5528 
 5529 operand vRegD_V27()
 5530 %{
 5531   constraint(ALLOC_IN_RC(v27_reg));
 5532   match(RegD);
 5533   op_cost(0);
 5534   format %{ %}
 5535   interface(REG_INTER);
 5536 %}
 5537 
 5538 operand vRegD_V28()
 5539 %{
 5540   constraint(ALLOC_IN_RC(v28_reg));
 5541   match(RegD);
 5542   op_cost(0);
 5543   format %{ %}
 5544   interface(REG_INTER);
 5545 %}
 5546 
 5547 operand vRegD_V29()
 5548 %{
 5549   constraint(ALLOC_IN_RC(v29_reg));
 5550   match(RegD);
 5551   op_cost(0);
 5552   format %{ %}
 5553   interface(REG_INTER);
 5554 %}
 5555 
 5556 operand vRegD_V30()
 5557 %{
 5558   constraint(ALLOC_IN_RC(v30_reg));
 5559   match(RegD);
 5560   op_cost(0);
 5561   format %{ %}
 5562   interface(REG_INTER);
 5563 %}
 5564 
 5565 operand vRegD_V31()
 5566 %{
 5567   constraint(ALLOC_IN_RC(v31_reg));
 5568   match(RegD);
 5569   op_cost(0);
 5570   format %{ %}
 5571   interface(REG_INTER);
 5572 %}
 5573 
 5574 operand pRegGov()
 5575 %{
 5576   constraint(ALLOC_IN_RC(gov_pr));
 5577   match(RegVectMask);
 5578   op_cost(0);
 5579   format %{ %}
 5580   interface(REG_INTER);
 5581 %}
 5582 
 5583 // Flags register, used as output of signed compare instructions
 5584 
 5585 // note that on AArch64 we also use this register as the output for
 5586 // for floating point compare instructions (CmpF CmpD). this ensures
 5587 // that ordered inequality tests use GT, GE, LT or LE none of which
 5588 // pass through cases where the result is unordered i.e. one or both
 5589 // inputs to the compare is a NaN. this means that the ideal code can
 5590 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5591 // (where the comparison should always fail). EQ and NE tests are
 5592 // always generated in ideal code so that unordered folds into the NE
 5593 // case, matching the behaviour of AArch64 NE.
 5594 //
 5595 // This differs from x86 where the outputs of FP compares use a
 5596 // special FP flags registers and where compares based on this
 5597 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5598 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5599 // to explicitly handle the unordered case in branches. x86 also has
 5600 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5601 
 5602 operand rFlagsReg()
 5603 %{
 5604   constraint(ALLOC_IN_RC(int_flags));
 5605   match(RegFlags);
 5606 
 5607   op_cost(0);
 5608   format %{ "RFLAGS" %}
 5609   interface(REG_INTER);
 5610 %}
 5611 
 5612 // Flags register, used as output of unsigned compare instructions
 5613 operand rFlagsRegU()
 5614 %{
 5615   constraint(ALLOC_IN_RC(int_flags));
 5616   match(RegFlags);
 5617 
 5618   op_cost(0);
 5619   format %{ "RFLAGSU" %}
 5620   interface(REG_INTER);
 5621 %}
 5622 
 5623 // Special Registers
 5624 
 5625 // Method Register
 5626 operand inline_cache_RegP(iRegP reg)
 5627 %{
 5628   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5629   match(reg);
 5630   match(iRegPNoSp);
 5631   op_cost(0);
 5632   format %{ %}
 5633   interface(REG_INTER);
 5634 %}
 5635 
 5636 // Thread Register
 5637 operand thread_RegP(iRegP reg)
 5638 %{
 5639   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5640   match(reg);
 5641   op_cost(0);
 5642   format %{ %}
 5643   interface(REG_INTER);
 5644 %}
 5645 
 5646 operand lr_RegP(iRegP reg)
 5647 %{
 5648   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5649   match(reg);
 5650   op_cost(0);
 5651   format %{ %}
 5652   interface(REG_INTER);
 5653 %}
 5654 
 5655 //----------Memory Operands----------------------------------------------------
 5656 
 5657 operand indirect(iRegP reg)
 5658 %{
 5659   constraint(ALLOC_IN_RC(ptr_reg));
 5660   match(reg);
 5661   op_cost(0);
 5662   format %{ "[$reg]" %}
 5663   interface(MEMORY_INTER) %{
 5664     base($reg);
 5665     index(0xffffffff);
 5666     scale(0x0);
 5667     disp(0x0);
 5668   %}
 5669 %}
 5670 
 5671 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5672 %{
 5673   constraint(ALLOC_IN_RC(ptr_reg));
 5674   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5675   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5676   op_cost(0);
 5677   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5678   interface(MEMORY_INTER) %{
 5679     base($reg);
 5680     index($ireg);
 5681     scale($scale);
 5682     disp(0x0);
 5683   %}
 5684 %}
 5685 
 5686 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5687 %{
 5688   constraint(ALLOC_IN_RC(ptr_reg));
 5689   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5690   match(AddP reg (LShiftL lreg scale));
 5691   op_cost(0);
 5692   format %{ "$reg, $lreg lsl($scale)" %}
 5693   interface(MEMORY_INTER) %{
 5694     base($reg);
 5695     index($lreg);
 5696     scale($scale);
 5697     disp(0x0);
 5698   %}
 5699 %}
 5700 
 5701 operand indIndexI2L(iRegP reg, iRegI ireg)
 5702 %{
 5703   constraint(ALLOC_IN_RC(ptr_reg));
 5704   match(AddP reg (ConvI2L ireg));
 5705   op_cost(0);
 5706   format %{ "$reg, $ireg, 0, I2L" %}
 5707   interface(MEMORY_INTER) %{
 5708     base($reg);
 5709     index($ireg);
 5710     scale(0x0);
 5711     disp(0x0);
 5712   %}
 5713 %}
 5714 
 5715 operand indIndex(iRegP reg, iRegL lreg)
 5716 %{
 5717   constraint(ALLOC_IN_RC(ptr_reg));
 5718   match(AddP reg lreg);
 5719   op_cost(0);
 5720   format %{ "$reg, $lreg" %}
 5721   interface(MEMORY_INTER) %{
 5722     base($reg);
 5723     index($lreg);
 5724     scale(0x0);
 5725     disp(0x0);
 5726   %}
 5727 %}
 5728 
 5729 operand indOffI(iRegP reg, immIOffset off)
 5730 %{
 5731   constraint(ALLOC_IN_RC(ptr_reg));
 5732   match(AddP reg off);
 5733   op_cost(0);
 5734   format %{ "[$reg, $off]" %}
 5735   interface(MEMORY_INTER) %{
 5736     base($reg);
 5737     index(0xffffffff);
 5738     scale(0x0);
 5739     disp($off);
 5740   %}
 5741 %}
 5742 
 5743 operand indOffI1(iRegP reg, immIOffset1 off)
 5744 %{
 5745   constraint(ALLOC_IN_RC(ptr_reg));
 5746   match(AddP reg off);
 5747   op_cost(0);
 5748   format %{ "[$reg, $off]" %}
 5749   interface(MEMORY_INTER) %{
 5750     base($reg);
 5751     index(0xffffffff);
 5752     scale(0x0);
 5753     disp($off);
 5754   %}
 5755 %}
 5756 
 5757 operand indOffI2(iRegP reg, immIOffset2 off)
 5758 %{
 5759   constraint(ALLOC_IN_RC(ptr_reg));
 5760   match(AddP reg off);
 5761   op_cost(0);
 5762   format %{ "[$reg, $off]" %}
 5763   interface(MEMORY_INTER) %{
 5764     base($reg);
 5765     index(0xffffffff);
 5766     scale(0x0);
 5767     disp($off);
 5768   %}
 5769 %}
 5770 
 5771 operand indOffI4(iRegP reg, immIOffset4 off)
 5772 %{
 5773   constraint(ALLOC_IN_RC(ptr_reg));
 5774   match(AddP reg off);
 5775   op_cost(0);
 5776   format %{ "[$reg, $off]" %}
 5777   interface(MEMORY_INTER) %{
 5778     base($reg);
 5779     index(0xffffffff);
 5780     scale(0x0);
 5781     disp($off);
 5782   %}
 5783 %}
 5784 
 5785 operand indOffI8(iRegP reg, immIOffset8 off)
 5786 %{
 5787   constraint(ALLOC_IN_RC(ptr_reg));
 5788   match(AddP reg off);
 5789   op_cost(0);
 5790   format %{ "[$reg, $off]" %}
 5791   interface(MEMORY_INTER) %{
 5792     base($reg);
 5793     index(0xffffffff);
 5794     scale(0x0);
 5795     disp($off);
 5796   %}
 5797 %}
 5798 
 5799 operand indOffI16(iRegP reg, immIOffset16 off)
 5800 %{
 5801   constraint(ALLOC_IN_RC(ptr_reg));
 5802   match(AddP reg off);
 5803   op_cost(0);
 5804   format %{ "[$reg, $off]" %}
 5805   interface(MEMORY_INTER) %{
 5806     base($reg);
 5807     index(0xffffffff);
 5808     scale(0x0);
 5809     disp($off);
 5810   %}
 5811 %}
 5812 
 5813 operand indOffL(iRegP reg, immLoffset off)
 5814 %{
 5815   constraint(ALLOC_IN_RC(ptr_reg));
 5816   match(AddP reg off);
 5817   op_cost(0);
 5818   format %{ "[$reg, $off]" %}
 5819   interface(MEMORY_INTER) %{
 5820     base($reg);
 5821     index(0xffffffff);
 5822     scale(0x0);
 5823     disp($off);
 5824   %}
 5825 %}
 5826 
 5827 operand indOffL1(iRegP reg, immLoffset1 off)
 5828 %{
 5829   constraint(ALLOC_IN_RC(ptr_reg));
 5830   match(AddP reg off);
 5831   op_cost(0);
 5832   format %{ "[$reg, $off]" %}
 5833   interface(MEMORY_INTER) %{
 5834     base($reg);
 5835     index(0xffffffff);
 5836     scale(0x0);
 5837     disp($off);
 5838   %}
 5839 %}
 5840 
 5841 operand indOffL2(iRegP reg, immLoffset2 off)
 5842 %{
 5843   constraint(ALLOC_IN_RC(ptr_reg));
 5844   match(AddP reg off);
 5845   op_cost(0);
 5846   format %{ "[$reg, $off]" %}
 5847   interface(MEMORY_INTER) %{
 5848     base($reg);
 5849     index(0xffffffff);
 5850     scale(0x0);
 5851     disp($off);
 5852   %}
 5853 %}
 5854 
 5855 operand indOffL4(iRegP reg, immLoffset4 off)
 5856 %{
 5857   constraint(ALLOC_IN_RC(ptr_reg));
 5858   match(AddP reg off);
 5859   op_cost(0);
 5860   format %{ "[$reg, $off]" %}
 5861   interface(MEMORY_INTER) %{
 5862     base($reg);
 5863     index(0xffffffff);
 5864     scale(0x0);
 5865     disp($off);
 5866   %}
 5867 %}
 5868 
 5869 operand indOffL8(iRegP reg, immLoffset8 off)
 5870 %{
 5871   constraint(ALLOC_IN_RC(ptr_reg));
 5872   match(AddP reg off);
 5873   op_cost(0);
 5874   format %{ "[$reg, $off]" %}
 5875   interface(MEMORY_INTER) %{
 5876     base($reg);
 5877     index(0xffffffff);
 5878     scale(0x0);
 5879     disp($off);
 5880   %}
 5881 %}
 5882 
 5883 operand indOffL16(iRegP reg, immLoffset16 off)
 5884 %{
 5885   constraint(ALLOC_IN_RC(ptr_reg));
 5886   match(AddP reg off);
 5887   op_cost(0);
 5888   format %{ "[$reg, $off]" %}
 5889   interface(MEMORY_INTER) %{
 5890     base($reg);
 5891     index(0xffffffff);
 5892     scale(0x0);
 5893     disp($off);
 5894   %}
 5895 %}
 5896 
 5897 operand indirectN(iRegN reg)
 5898 %{
 5899   predicate(CompressedOops::shift() == 0);
 5900   constraint(ALLOC_IN_RC(ptr_reg));
 5901   match(DecodeN reg);
 5902   op_cost(0);
 5903   format %{ "[$reg]\t# narrow" %}
 5904   interface(MEMORY_INTER) %{
 5905     base($reg);
 5906     index(0xffffffff);
 5907     scale(0x0);
 5908     disp(0x0);
 5909   %}
 5910 %}
 5911 
 5912 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5913 %{
 5914   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5915   constraint(ALLOC_IN_RC(ptr_reg));
 5916   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5917   op_cost(0);
 5918   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5919   interface(MEMORY_INTER) %{
 5920     base($reg);
 5921     index($ireg);
 5922     scale($scale);
 5923     disp(0x0);
 5924   %}
 5925 %}
 5926 
 5927 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5928 %{
 5929   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5930   constraint(ALLOC_IN_RC(ptr_reg));
 5931   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5932   op_cost(0);
 5933   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5934   interface(MEMORY_INTER) %{
 5935     base($reg);
 5936     index($lreg);
 5937     scale($scale);
 5938     disp(0x0);
 5939   %}
 5940 %}
 5941 
 5942 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5943 %{
 5944   predicate(CompressedOops::shift() == 0);
 5945   constraint(ALLOC_IN_RC(ptr_reg));
 5946   match(AddP (DecodeN reg) (ConvI2L ireg));
 5947   op_cost(0);
 5948   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5949   interface(MEMORY_INTER) %{
 5950     base($reg);
 5951     index($ireg);
 5952     scale(0x0);
 5953     disp(0x0);
 5954   %}
 5955 %}
 5956 
 5957 operand indIndexN(iRegN reg, iRegL lreg)
 5958 %{
 5959   predicate(CompressedOops::shift() == 0);
 5960   constraint(ALLOC_IN_RC(ptr_reg));
 5961   match(AddP (DecodeN reg) lreg);
 5962   op_cost(0);
 5963   format %{ "$reg, $lreg\t# narrow" %}
 5964   interface(MEMORY_INTER) %{
 5965     base($reg);
 5966     index($lreg);
 5967     scale(0x0);
 5968     disp(0x0);
 5969   %}
 5970 %}
 5971 
 5972 operand indOffIN(iRegN reg, immIOffset off)
 5973 %{
 5974   predicate(CompressedOops::shift() == 0);
 5975   constraint(ALLOC_IN_RC(ptr_reg));
 5976   match(AddP (DecodeN reg) off);
 5977   op_cost(0);
 5978   format %{ "[$reg, $off]\t# narrow" %}
 5979   interface(MEMORY_INTER) %{
 5980     base($reg);
 5981     index(0xffffffff);
 5982     scale(0x0);
 5983     disp($off);
 5984   %}
 5985 %}
 5986 
 5987 operand indOffLN(iRegN reg, immLoffset off)
 5988 %{
 5989   predicate(CompressedOops::shift() == 0);
 5990   constraint(ALLOC_IN_RC(ptr_reg));
 5991   match(AddP (DecodeN reg) off);
 5992   op_cost(0);
 5993   format %{ "[$reg, $off]\t# narrow" %}
 5994   interface(MEMORY_INTER) %{
 5995     base($reg);
 5996     index(0xffffffff);
 5997     scale(0x0);
 5998     disp($off);
 5999   %}
 6000 %}
 6001 
 6002 
 6003 
 6004 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6005 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6006 %{
 6007   constraint(ALLOC_IN_RC(ptr_reg));
 6008   match(AddP reg off);
 6009   op_cost(0);
 6010   format %{ "[$reg, $off]" %}
 6011   interface(MEMORY_INTER) %{
 6012     base($reg);
 6013     index(0xffffffff);
 6014     scale(0x0);
 6015     disp($off);
 6016   %}
 6017 %}
 6018 
 6019 //----------Special Memory Operands--------------------------------------------
 6020 // Stack Slot Operand - This operand is used for loading and storing temporary
 6021 //                      values on the stack where a match requires a value to
 6022 //                      flow through memory.
 6023 operand stackSlotP(sRegP reg)
 6024 %{
 6025   constraint(ALLOC_IN_RC(stack_slots));
 6026   op_cost(100);
 6027   // No match rule because this operand is only generated in matching
 6028   // match(RegP);
 6029   format %{ "[$reg]" %}
 6030   interface(MEMORY_INTER) %{
 6031     base(0x1e);  // RSP
 6032     index(0x0);  // No Index
 6033     scale(0x0);  // No Scale
 6034     disp($reg);  // Stack Offset
 6035   %}
 6036 %}
 6037 
 6038 operand stackSlotI(sRegI reg)
 6039 %{
 6040   constraint(ALLOC_IN_RC(stack_slots));
 6041   // No match rule because this operand is only generated in matching
 6042   // match(RegI);
 6043   format %{ "[$reg]" %}
 6044   interface(MEMORY_INTER) %{
 6045     base(0x1e);  // RSP
 6046     index(0x0);  // No Index
 6047     scale(0x0);  // No Scale
 6048     disp($reg);  // Stack Offset
 6049   %}
 6050 %}
 6051 
 6052 operand stackSlotF(sRegF reg)
 6053 %{
 6054   constraint(ALLOC_IN_RC(stack_slots));
 6055   // No match rule because this operand is only generated in matching
 6056   // match(RegF);
 6057   format %{ "[$reg]" %}
 6058   interface(MEMORY_INTER) %{
 6059     base(0x1e);  // RSP
 6060     index(0x0);  // No Index
 6061     scale(0x0);  // No Scale
 6062     disp($reg);  // Stack Offset
 6063   %}
 6064 %}
 6065 
 6066 operand stackSlotD(sRegD reg)
 6067 %{
 6068   constraint(ALLOC_IN_RC(stack_slots));
 6069   // No match rule because this operand is only generated in matching
 6070   // match(RegD);
 6071   format %{ "[$reg]" %}
 6072   interface(MEMORY_INTER) %{
 6073     base(0x1e);  // RSP
 6074     index(0x0);  // No Index
 6075     scale(0x0);  // No Scale
 6076     disp($reg);  // Stack Offset
 6077   %}
 6078 %}
 6079 
 6080 operand stackSlotL(sRegL reg)
 6081 %{
 6082   constraint(ALLOC_IN_RC(stack_slots));
 6083   // No match rule because this operand is only generated in matching
 6084   // match(RegL);
 6085   format %{ "[$reg]" %}
 6086   interface(MEMORY_INTER) %{
 6087     base(0x1e);  // RSP
 6088     index(0x0);  // No Index
 6089     scale(0x0);  // No Scale
 6090     disp($reg);  // Stack Offset
 6091   %}
 6092 %}
 6093 
 6094 // Operands for expressing Control Flow
 6095 // NOTE: Label is a predefined operand which should not be redefined in
 6096 //       the AD file. It is generically handled within the ADLC.
 6097 
 6098 //----------Conditional Branch Operands----------------------------------------
 6099 // Comparison Op  - This is the operation of the comparison, and is limited to
 6100 //                  the following set of codes:
 6101 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6102 //
 6103 // Other attributes of the comparison, such as unsignedness, are specified
 6104 // by the comparison instruction that sets a condition code flags register.
 6105 // That result is represented by a flags operand whose subtype is appropriate
 6106 // to the unsignedness (etc.) of the comparison.
 6107 //
 6108 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6109 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6110 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6111 
 6112 // used for signed integral comparisons and fp comparisons
 6113 
 6114 operand cmpOp()
 6115 %{
 6116   match(Bool);
 6117 
 6118   format %{ "" %}
 6119   interface(COND_INTER) %{
 6120     equal(0x0, "eq");
 6121     not_equal(0x1, "ne");
 6122     less(0xb, "lt");
 6123     greater_equal(0xa, "ge");
 6124     less_equal(0xd, "le");
 6125     greater(0xc, "gt");
 6126     overflow(0x6, "vs");
 6127     no_overflow(0x7, "vc");
 6128   %}
 6129 %}
 6130 
 6131 // used for unsigned integral comparisons
 6132 
 6133 operand cmpOpU()
 6134 %{
 6135   match(Bool);
 6136 
 6137   format %{ "" %}
 6138   interface(COND_INTER) %{
 6139     equal(0x0, "eq");
 6140     not_equal(0x1, "ne");
 6141     less(0x3, "lo");
 6142     greater_equal(0x2, "hs");
 6143     less_equal(0x9, "ls");
 6144     greater(0x8, "hi");
 6145     overflow(0x6, "vs");
 6146     no_overflow(0x7, "vc");
 6147   %}
 6148 %}
 6149 
 6150 // used for certain integral comparisons which can be
 6151 // converted to cbxx or tbxx instructions
 6152 
 6153 operand cmpOpEqNe()
 6154 %{
 6155   match(Bool);
 6156   op_cost(0);
 6157   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6158             || n->as_Bool()->_test._test == BoolTest::eq);
 6159 
 6160   format %{ "" %}
 6161   interface(COND_INTER) %{
 6162     equal(0x0, "eq");
 6163     not_equal(0x1, "ne");
 6164     less(0xb, "lt");
 6165     greater_equal(0xa, "ge");
 6166     less_equal(0xd, "le");
 6167     greater(0xc, "gt");
 6168     overflow(0x6, "vs");
 6169     no_overflow(0x7, "vc");
 6170   %}
 6171 %}
 6172 
 6173 // used for certain integral comparisons which can be
 6174 // converted to cbxx or tbxx instructions
 6175 
 6176 operand cmpOpLtGe()
 6177 %{
 6178   match(Bool);
 6179   op_cost(0);
 6180 
 6181   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6182             || n->as_Bool()->_test._test == BoolTest::ge);
 6183 
 6184   format %{ "" %}
 6185   interface(COND_INTER) %{
 6186     equal(0x0, "eq");
 6187     not_equal(0x1, "ne");
 6188     less(0xb, "lt");
 6189     greater_equal(0xa, "ge");
 6190     less_equal(0xd, "le");
 6191     greater(0xc, "gt");
 6192     overflow(0x6, "vs");
 6193     no_overflow(0x7, "vc");
 6194   %}
 6195 %}
 6196 
 6197 // used for certain unsigned integral comparisons which can be
 6198 // converted to cbxx or tbxx instructions
 6199 
 6200 operand cmpOpUEqNeLtGe()
 6201 %{
 6202   match(Bool);
 6203   op_cost(0);
 6204 
 6205   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6206             || n->as_Bool()->_test._test == BoolTest::ne
 6207             || n->as_Bool()->_test._test == BoolTest::lt
 6208             || n->as_Bool()->_test._test == BoolTest::ge);
 6209 
 6210   format %{ "" %}
 6211   interface(COND_INTER) %{
 6212     equal(0x0, "eq");
 6213     not_equal(0x1, "ne");
 6214     less(0xb, "lt");
 6215     greater_equal(0xa, "ge");
 6216     less_equal(0xd, "le");
 6217     greater(0xc, "gt");
 6218     overflow(0x6, "vs");
 6219     no_overflow(0x7, "vc");
 6220   %}
 6221 %}
 6222 
 6223 // Special operand allowing long args to int ops to be truncated for free
 6224 
 6225 operand iRegL2I(iRegL reg) %{
 6226 
 6227   op_cost(0);
 6228 
 6229   match(ConvL2I reg);
 6230 
 6231   format %{ "l2i($reg)" %}
 6232 
 6233   interface(REG_INTER)
 6234 %}
 6235 
 6236 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6237 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6238 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6239 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6240 
 6241 //----------OPERAND CLASSES----------------------------------------------------
 6242 // Operand Classes are groups of operands that are used as to simplify
 6243 // instruction definitions by not requiring the AD writer to specify
 6244 // separate instructions for every form of operand when the
 6245 // instruction accepts multiple operand types with the same basic
 6246 // encoding and format. The classic case of this is memory operands.
 6247 
 6248 // memory is used to define read/write location for load/store
 6249 // instruction defs. we can turn a memory op into an Address
 6250 
 6251 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6252                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6253 
 6254 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6255                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6256 
 6257 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6258                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6259 
 6260 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6261                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6262 
 6263 // All of the memory operands. For the pipeline description.
 6264 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6265                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6266                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6267 
 6268 
 6269 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6270 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6271 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6272 // can be elided because the 32-bit instruction will just employ the
 6273 // lower 32 bits anyway.
 6274 //
 6275 // n.b. this does not elide all L2I conversions. if the truncated
 6276 // value is consumed by more than one operation then the ConvL2I
 6277 // cannot be bundled into the consuming nodes so an l2i gets planted
 6278 // (actually a movw $dst $src) and the downstream instructions consume
 6279 // the result of the l2i as an iRegI input. That's a shame since the
 6280 // movw is actually redundant but its not too costly.
 6281 
 6282 opclass iRegIorL2I(iRegI, iRegL2I);
 6283 
 6284 //----------PIPELINE-----------------------------------------------------------
 6285 // Rules which define the behavior of the target architectures pipeline.
 6286 
 6287 // For specific pipelines, eg A53, define the stages of that pipeline
 6288 //pipe_desc(ISS, EX1, EX2, WR);
 6289 #define ISS S0
 6290 #define EX1 S1
 6291 #define EX2 S2
 6292 #define WR  S3
 6293 
 6294 // Integer ALU reg operation
 6295 pipeline %{
 6296 
 6297 attributes %{
 6298   // ARM instructions are of fixed length
 6299   fixed_size_instructions;        // Fixed size instructions TODO does
 6300   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6301   // ARM instructions come in 32-bit word units
 6302   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6303   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6304   instruction_fetch_units = 1;       // of 64 bytes
 6305 
 6306   // List of nop instructions
 6307   nops( MachNop );
 6308 %}
 6309 
 6310 // We don't use an actual pipeline model so don't care about resources
 6311 // or description. we do use pipeline classes to introduce fixed
 6312 // latencies
 6313 
 6314 //----------RESOURCES----------------------------------------------------------
 6315 // Resources are the functional units available to the machine
 6316 
 6317 resources( INS0, INS1, INS01 = INS0 | INS1,
 6318            ALU0, ALU1, ALU = ALU0 | ALU1,
 6319            MAC,
 6320            DIV,
 6321            BRANCH,
 6322            LDST,
 6323            NEON_FP);
 6324 
 6325 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6326 // Pipeline Description specifies the stages in the machine's pipeline
 6327 
 6328 // Define the pipeline as a generic 6 stage pipeline
 6329 pipe_desc(S0, S1, S2, S3, S4, S5);
 6330 
 6331 //----------PIPELINE CLASSES---------------------------------------------------
 6332 // Pipeline Classes describe the stages in which input and output are
 6333 // referenced by the hardware pipeline.
 6334 
 6335 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6336 %{
 6337   single_instruction;
 6338   src1   : S1(read);
 6339   src2   : S2(read);
 6340   dst    : S5(write);
 6341   INS01  : ISS;
 6342   NEON_FP : S5;
 6343 %}
 6344 
 6345 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6346 %{
 6347   single_instruction;
 6348   src1   : S1(read);
 6349   src2   : S2(read);
 6350   dst    : S5(write);
 6351   INS01  : ISS;
 6352   NEON_FP : S5;
 6353 %}
 6354 
 6355 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6356 %{
 6357   single_instruction;
 6358   src    : S1(read);
 6359   dst    : S5(write);
 6360   INS01  : ISS;
 6361   NEON_FP : S5;
 6362 %}
 6363 
 6364 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6365 %{
 6366   single_instruction;
 6367   src    : S1(read);
 6368   dst    : S5(write);
 6369   INS01  : ISS;
 6370   NEON_FP : S5;
 6371 %}
 6372 
 6373 pipe_class fp_d2f(vRegF dst, vRegD src)
 6374 %{
 6375   single_instruction;
 6376   src    : S1(read);
 6377   dst    : S5(write);
 6378   INS01  : ISS;
 6379   NEON_FP : S5;
 6380 %}
 6381 
 6382 pipe_class fp_f2d(vRegD dst, vRegF src)
 6383 %{
 6384   single_instruction;
 6385   src    : S1(read);
 6386   dst    : S5(write);
 6387   INS01  : ISS;
 6388   NEON_FP : S5;
 6389 %}
 6390 
 6391 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6392 %{
 6393   single_instruction;
 6394   src    : S1(read);
 6395   dst    : S5(write);
 6396   INS01  : ISS;
 6397   NEON_FP : S5;
 6398 %}
 6399 
 6400 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6401 %{
 6402   single_instruction;
 6403   src    : S1(read);
 6404   dst    : S5(write);
 6405   INS01  : ISS;
 6406   NEON_FP : S5;
 6407 %}
 6408 
 6409 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6410 %{
 6411   single_instruction;
 6412   src    : S1(read);
 6413   dst    : S5(write);
 6414   INS01  : ISS;
 6415   NEON_FP : S5;
 6416 %}
 6417 
 6418 pipe_class fp_l2f(vRegF dst, iRegL src)
 6419 %{
 6420   single_instruction;
 6421   src    : S1(read);
 6422   dst    : S5(write);
 6423   INS01  : ISS;
 6424   NEON_FP : S5;
 6425 %}
 6426 
 6427 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6428 %{
 6429   single_instruction;
 6430   src    : S1(read);
 6431   dst    : S5(write);
 6432   INS01  : ISS;
 6433   NEON_FP : S5;
 6434 %}
 6435 
 6436 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6437 %{
 6438   single_instruction;
 6439   src    : S1(read);
 6440   dst    : S5(write);
 6441   INS01  : ISS;
 6442   NEON_FP : S5;
 6443 %}
 6444 
 6445 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6446 %{
 6447   single_instruction;
 6448   src    : S1(read);
 6449   dst    : S5(write);
 6450   INS01  : ISS;
 6451   NEON_FP : S5;
 6452 %}
 6453 
 6454 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6455 %{
 6456   single_instruction;
 6457   src    : S1(read);
 6458   dst    : S5(write);
 6459   INS01  : ISS;
 6460   NEON_FP : S5;
 6461 %}
 6462 
 6463 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6464 %{
 6465   single_instruction;
 6466   src1   : S1(read);
 6467   src2   : S2(read);
 6468   dst    : S5(write);
 6469   INS0   : ISS;
 6470   NEON_FP : S5;
 6471 %}
 6472 
 6473 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6474 %{
 6475   single_instruction;
 6476   src1   : S1(read);
 6477   src2   : S2(read);
 6478   dst    : S5(write);
 6479   INS0   : ISS;
 6480   NEON_FP : S5;
 6481 %}
 6482 
 6483 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6484 %{
 6485   single_instruction;
 6486   cr     : S1(read);
 6487   src1   : S1(read);
 6488   src2   : S1(read);
 6489   dst    : S3(write);
 6490   INS01  : ISS;
 6491   NEON_FP : S3;
 6492 %}
 6493 
 6494 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6495 %{
 6496   single_instruction;
 6497   cr     : S1(read);
 6498   src1   : S1(read);
 6499   src2   : S1(read);
 6500   dst    : S3(write);
 6501   INS01  : ISS;
 6502   NEON_FP : S3;
 6503 %}
 6504 
 6505 pipe_class fp_imm_s(vRegF dst)
 6506 %{
 6507   single_instruction;
 6508   dst    : S3(write);
 6509   INS01  : ISS;
 6510   NEON_FP : S3;
 6511 %}
 6512 
 6513 pipe_class fp_imm_d(vRegD dst)
 6514 %{
 6515   single_instruction;
 6516   dst    : S3(write);
 6517   INS01  : ISS;
 6518   NEON_FP : S3;
 6519 %}
 6520 
 6521 pipe_class fp_load_constant_s(vRegF dst)
 6522 %{
 6523   single_instruction;
 6524   dst    : S4(write);
 6525   INS01  : ISS;
 6526   NEON_FP : S4;
 6527 %}
 6528 
 6529 pipe_class fp_load_constant_d(vRegD dst)
 6530 %{
 6531   single_instruction;
 6532   dst    : S4(write);
 6533   INS01  : ISS;
 6534   NEON_FP : S4;
 6535 %}
 6536 
 6537 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6538 %{
 6539   single_instruction;
 6540   dst    : S5(write);
 6541   src1   : S1(read);
 6542   src2   : S1(read);
 6543   INS01  : ISS;
 6544   NEON_FP : S5;
 6545 %}
 6546 
 6547 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6548 %{
 6549   single_instruction;
 6550   dst    : S5(write);
 6551   src1   : S1(read);
 6552   src2   : S1(read);
 6553   INS0   : ISS;
 6554   NEON_FP : S5;
 6555 %}
 6556 
 6557 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6558 %{
 6559   single_instruction;
 6560   dst    : S5(write);
 6561   src1   : S1(read);
 6562   src2   : S1(read);
 6563   dst    : S1(read);
 6564   INS01  : ISS;
 6565   NEON_FP : S5;
 6566 %}
 6567 
 6568 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6569 %{
 6570   single_instruction;
 6571   dst    : S5(write);
 6572   src1   : S1(read);
 6573   src2   : S1(read);
 6574   dst    : S1(read);
 6575   INS0   : ISS;
 6576   NEON_FP : S5;
 6577 %}
 6578 
 6579 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6580 %{
 6581   single_instruction;
 6582   dst    : S4(write);
 6583   src1   : S2(read);
 6584   src2   : S2(read);
 6585   INS01  : ISS;
 6586   NEON_FP : S4;
 6587 %}
 6588 
 6589 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6590 %{
 6591   single_instruction;
 6592   dst    : S4(write);
 6593   src1   : S2(read);
 6594   src2   : S2(read);
 6595   INS0   : ISS;
 6596   NEON_FP : S4;
 6597 %}
 6598 
 6599 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6600 %{
 6601   single_instruction;
 6602   dst    : S3(write);
 6603   src1   : S2(read);
 6604   src2   : S2(read);
 6605   INS01  : ISS;
 6606   NEON_FP : S3;
 6607 %}
 6608 
 6609 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6610 %{
 6611   single_instruction;
 6612   dst    : S3(write);
 6613   src1   : S2(read);
 6614   src2   : S2(read);
 6615   INS0   : ISS;
 6616   NEON_FP : S3;
 6617 %}
 6618 
 6619 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6620 %{
 6621   single_instruction;
 6622   dst    : S3(write);
 6623   src    : S1(read);
 6624   shift  : S1(read);
 6625   INS01  : ISS;
 6626   NEON_FP : S3;
 6627 %}
 6628 
 6629 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6630 %{
 6631   single_instruction;
 6632   dst    : S3(write);
 6633   src    : S1(read);
 6634   shift  : S1(read);
 6635   INS0   : ISS;
 6636   NEON_FP : S3;
 6637 %}
 6638 
 6639 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6640 %{
 6641   single_instruction;
 6642   dst    : S3(write);
 6643   src    : S1(read);
 6644   INS01  : ISS;
 6645   NEON_FP : S3;
 6646 %}
 6647 
 6648 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6649 %{
 6650   single_instruction;
 6651   dst    : S3(write);
 6652   src    : S1(read);
 6653   INS0   : ISS;
 6654   NEON_FP : S3;
 6655 %}
 6656 
 6657 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6658 %{
 6659   single_instruction;
 6660   dst    : S5(write);
 6661   src1   : S1(read);
 6662   src2   : S1(read);
 6663   INS01  : ISS;
 6664   NEON_FP : S5;
 6665 %}
 6666 
 6667 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6668 %{
 6669   single_instruction;
 6670   dst    : S5(write);
 6671   src1   : S1(read);
 6672   src2   : S1(read);
 6673   INS0   : ISS;
 6674   NEON_FP : S5;
 6675 %}
 6676 
 6677 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6678 %{
 6679   single_instruction;
 6680   dst    : S5(write);
 6681   src1   : S1(read);
 6682   src2   : S1(read);
 6683   INS0   : ISS;
 6684   NEON_FP : S5;
 6685 %}
 6686 
 6687 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6688 %{
 6689   single_instruction;
 6690   dst    : S5(write);
 6691   src1   : S1(read);
 6692   src2   : S1(read);
 6693   INS0   : ISS;
 6694   NEON_FP : S5;
 6695 %}
 6696 
 6697 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6698 %{
 6699   single_instruction;
 6700   dst    : S5(write);
 6701   src    : S1(read);
 6702   INS0   : ISS;
 6703   NEON_FP : S5;
 6704 %}
 6705 
 6706 pipe_class vunop_fp64(vecD dst, vecD src)
 6707 %{
 6708   single_instruction;
 6709   dst    : S5(write);
 6710   src    : S1(read);
 6711   INS01  : ISS;
 6712   NEON_FP : S5;
 6713 %}
 6714 
 6715 pipe_class vunop_fp128(vecX dst, vecX src)
 6716 %{
 6717   single_instruction;
 6718   dst    : S5(write);
 6719   src    : S1(read);
 6720   INS0   : ISS;
 6721   NEON_FP : S5;
 6722 %}
 6723 
 6724 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6725 %{
 6726   single_instruction;
 6727   dst    : S3(write);
 6728   src    : S1(read);
 6729   INS01  : ISS;
 6730   NEON_FP : S3;
 6731 %}
 6732 
 6733 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6734 %{
 6735   single_instruction;
 6736   dst    : S3(write);
 6737   src    : S1(read);
 6738   INS01  : ISS;
 6739   NEON_FP : S3;
 6740 %}
 6741 
 6742 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6743 %{
 6744   single_instruction;
 6745   dst    : S3(write);
 6746   src    : S1(read);
 6747   INS01  : ISS;
 6748   NEON_FP : S3;
 6749 %}
 6750 
 6751 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6752 %{
 6753   single_instruction;
 6754   dst    : S3(write);
 6755   src    : S1(read);
 6756   INS01  : ISS;
 6757   NEON_FP : S3;
 6758 %}
 6759 
 6760 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6761 %{
 6762   single_instruction;
 6763   dst    : S3(write);
 6764   src    : S1(read);
 6765   INS01  : ISS;
 6766   NEON_FP : S3;
 6767 %}
 6768 
 6769 pipe_class vmovi_reg_imm64(vecD dst)
 6770 %{
 6771   single_instruction;
 6772   dst    : S3(write);
 6773   INS01  : ISS;
 6774   NEON_FP : S3;
 6775 %}
 6776 
 6777 pipe_class vmovi_reg_imm128(vecX dst)
 6778 %{
 6779   single_instruction;
 6780   dst    : S3(write);
 6781   INS0   : ISS;
 6782   NEON_FP : S3;
 6783 %}
 6784 
 6785 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6786 %{
 6787   single_instruction;
 6788   dst    : S5(write);
 6789   mem    : ISS(read);
 6790   INS01  : ISS;
 6791   NEON_FP : S3;
 6792 %}
 6793 
 6794 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6795 %{
 6796   single_instruction;
 6797   dst    : S5(write);
 6798   mem    : ISS(read);
 6799   INS01  : ISS;
 6800   NEON_FP : S3;
 6801 %}
 6802 
 6803 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6804 %{
 6805   single_instruction;
 6806   mem    : ISS(read);
 6807   src    : S2(read);
 6808   INS01  : ISS;
 6809   NEON_FP : S3;
 6810 %}
 6811 
 6812 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6813 %{
 6814   single_instruction;
 6815   mem    : ISS(read);
 6816   src    : S2(read);
 6817   INS01  : ISS;
 6818   NEON_FP : S3;
 6819 %}
 6820 
 6821 //------- Integer ALU operations --------------------------
 6822 
 6823 // Integer ALU reg-reg operation
 6824 // Operands needed in EX1, result generated in EX2
 6825 // Eg.  ADD     x0, x1, x2
 6826 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6827 %{
 6828   single_instruction;
 6829   dst    : EX2(write);
 6830   src1   : EX1(read);
 6831   src2   : EX1(read);
 6832   INS01  : ISS; // Dual issue as instruction 0 or 1
 6833   ALU    : EX2;
 6834 %}
 6835 
 6836 // Integer ALU reg-reg operation with constant shift
 6837 // Shifted register must be available in LATE_ISS instead of EX1
 6838 // Eg.  ADD     x0, x1, x2, LSL #2
 6839 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6840 %{
 6841   single_instruction;
 6842   dst    : EX2(write);
 6843   src1   : EX1(read);
 6844   src2   : ISS(read);
 6845   INS01  : ISS;
 6846   ALU    : EX2;
 6847 %}
 6848 
 6849 // Integer ALU reg operation with constant shift
 6850 // Eg.  LSL     x0, x1, #shift
 6851 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6852 %{
 6853   single_instruction;
 6854   dst    : EX2(write);
 6855   src1   : ISS(read);
 6856   INS01  : ISS;
 6857   ALU    : EX2;
 6858 %}
 6859 
 6860 // Integer ALU reg-reg operation with variable shift
 6861 // Both operands must be available in LATE_ISS instead of EX1
 6862 // Result is available in EX1 instead of EX2
 6863 // Eg.  LSLV    x0, x1, x2
 6864 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6865 %{
 6866   single_instruction;
 6867   dst    : EX1(write);
 6868   src1   : ISS(read);
 6869   src2   : ISS(read);
 6870   INS01  : ISS;
 6871   ALU    : EX1;
 6872 %}
 6873 
 6874 // Integer ALU reg-reg operation with extract
 6875 // As for _vshift above, but result generated in EX2
 6876 // Eg.  EXTR    x0, x1, x2, #N
 6877 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6878 %{
 6879   single_instruction;
 6880   dst    : EX2(write);
 6881   src1   : ISS(read);
 6882   src2   : ISS(read);
 6883   INS1   : ISS; // Can only dual issue as Instruction 1
 6884   ALU    : EX1;
 6885 %}
 6886 
 6887 // Integer ALU reg operation
 6888 // Eg.  NEG     x0, x1
 6889 pipe_class ialu_reg(iRegI dst, iRegI src)
 6890 %{
 6891   single_instruction;
 6892   dst    : EX2(write);
 6893   src    : EX1(read);
 6894   INS01  : ISS;
 6895   ALU    : EX2;
 6896 %}
 6897 
 6898 // Integer ALU reg mmediate operation
 6899 // Eg.  ADD     x0, x1, #N
 6900 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6901 %{
 6902   single_instruction;
 6903   dst    : EX2(write);
 6904   src1   : EX1(read);
 6905   INS01  : ISS;
 6906   ALU    : EX2;
 6907 %}
 6908 
 6909 // Integer ALU immediate operation (no source operands)
 6910 // Eg.  MOV     x0, #N
 6911 pipe_class ialu_imm(iRegI dst)
 6912 %{
 6913   single_instruction;
 6914   dst    : EX1(write);
 6915   INS01  : ISS;
 6916   ALU    : EX1;
 6917 %}
 6918 
 6919 //------- Compare operation -------------------------------
 6920 
 6921 // Compare reg-reg
 6922 // Eg.  CMP     x0, x1
 6923 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6924 %{
 6925   single_instruction;
 6926 //  fixed_latency(16);
 6927   cr     : EX2(write);
 6928   op1    : EX1(read);
 6929   op2    : EX1(read);
 6930   INS01  : ISS;
 6931   ALU    : EX2;
 6932 %}
 6933 
 6934 // Compare reg-reg
 6935 // Eg.  CMP     x0, #N
 6936 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6937 %{
 6938   single_instruction;
 6939 //  fixed_latency(16);
 6940   cr     : EX2(write);
 6941   op1    : EX1(read);
 6942   INS01  : ISS;
 6943   ALU    : EX2;
 6944 %}
 6945 
 6946 //------- Conditional instructions ------------------------
 6947 
 6948 // Conditional no operands
 6949 // Eg.  CSINC   x0, zr, zr, <cond>
 6950 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6951 %{
 6952   single_instruction;
 6953   cr     : EX1(read);
 6954   dst    : EX2(write);
 6955   INS01  : ISS;
 6956   ALU    : EX2;
 6957 %}
 6958 
 6959 // Conditional 2 operand
 6960 // EG.  CSEL    X0, X1, X2, <cond>
 6961 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6962 %{
 6963   single_instruction;
 6964   cr     : EX1(read);
 6965   src1   : EX1(read);
 6966   src2   : EX1(read);
 6967   dst    : EX2(write);
 6968   INS01  : ISS;
 6969   ALU    : EX2;
 6970 %}
 6971 
 6972 // Conditional 2 operand
 6973 // EG.  CSEL    X0, X1, X2, <cond>
 6974 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6975 %{
 6976   single_instruction;
 6977   cr     : EX1(read);
 6978   src    : EX1(read);
 6979   dst    : EX2(write);
 6980   INS01  : ISS;
 6981   ALU    : EX2;
 6982 %}
 6983 
 6984 //------- Multiply pipeline operations --------------------
 6985 
 6986 // Multiply reg-reg
 6987 // Eg.  MUL     w0, w1, w2
 6988 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6989 %{
 6990   single_instruction;
 6991   dst    : WR(write);
 6992   src1   : ISS(read);
 6993   src2   : ISS(read);
 6994   INS01  : ISS;
 6995   MAC    : WR;
 6996 %}
 6997 
 6998 // Multiply accumulate
 6999 // Eg.  MADD    w0, w1, w2, w3
 7000 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7001 %{
 7002   single_instruction;
 7003   dst    : WR(write);
 7004   src1   : ISS(read);
 7005   src2   : ISS(read);
 7006   src3   : ISS(read);
 7007   INS01  : ISS;
 7008   MAC    : WR;
 7009 %}
 7010 
 7011 // Eg.  MUL     w0, w1, w2
 7012 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7013 %{
 7014   single_instruction;
 7015   fixed_latency(3); // Maximum latency for 64 bit mul
 7016   dst    : WR(write);
 7017   src1   : ISS(read);
 7018   src2   : ISS(read);
 7019   INS01  : ISS;
 7020   MAC    : WR;
 7021 %}
 7022 
 7023 // Multiply accumulate
 7024 // Eg.  MADD    w0, w1, w2, w3
 7025 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7026 %{
 7027   single_instruction;
 7028   fixed_latency(3); // Maximum latency for 64 bit mul
 7029   dst    : WR(write);
 7030   src1   : ISS(read);
 7031   src2   : ISS(read);
 7032   src3   : ISS(read);
 7033   INS01  : ISS;
 7034   MAC    : WR;
 7035 %}
 7036 
 7037 //------- Divide pipeline operations --------------------
 7038 
 7039 // Eg.  SDIV    w0, w1, w2
 7040 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7041 %{
 7042   single_instruction;
 7043   fixed_latency(8); // Maximum latency for 32 bit divide
 7044   dst    : WR(write);
 7045   src1   : ISS(read);
 7046   src2   : ISS(read);
 7047   INS0   : ISS; // Can only dual issue as instruction 0
 7048   DIV    : WR;
 7049 %}
 7050 
 7051 // Eg.  SDIV    x0, x1, x2
 7052 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7053 %{
 7054   single_instruction;
 7055   fixed_latency(16); // Maximum latency for 64 bit divide
 7056   dst    : WR(write);
 7057   src1   : ISS(read);
 7058   src2   : ISS(read);
 7059   INS0   : ISS; // Can only dual issue as instruction 0
 7060   DIV    : WR;
 7061 %}
 7062 
 7063 //------- Load pipeline operations ------------------------
 7064 
 7065 // Load - prefetch
 7066 // Eg.  PFRM    <mem>
 7067 pipe_class iload_prefetch(memory mem)
 7068 %{
 7069   single_instruction;
 7070   mem    : ISS(read);
 7071   INS01  : ISS;
 7072   LDST   : WR;
 7073 %}
 7074 
 7075 // Load - reg, mem
 7076 // Eg.  LDR     x0, <mem>
 7077 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7078 %{
 7079   single_instruction;
 7080   dst    : WR(write);
 7081   mem    : ISS(read);
 7082   INS01  : ISS;
 7083   LDST   : WR;
 7084 %}
 7085 
 7086 // Load - reg, reg
 7087 // Eg.  LDR     x0, [sp, x1]
 7088 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7089 %{
 7090   single_instruction;
 7091   dst    : WR(write);
 7092   src    : ISS(read);
 7093   INS01  : ISS;
 7094   LDST   : WR;
 7095 %}
 7096 
 7097 //------- Store pipeline operations -----------------------
 7098 
 7099 // Store - zr, mem
 7100 // Eg.  STR     zr, <mem>
 7101 pipe_class istore_mem(memory mem)
 7102 %{
 7103   single_instruction;
 7104   mem    : ISS(read);
 7105   INS01  : ISS;
 7106   LDST   : WR;
 7107 %}
 7108 
 7109 // Store - reg, mem
 7110 // Eg.  STR     x0, <mem>
 7111 pipe_class istore_reg_mem(iRegI src, memory mem)
 7112 %{
 7113   single_instruction;
 7114   mem    : ISS(read);
 7115   src    : EX2(read);
 7116   INS01  : ISS;
 7117   LDST   : WR;
 7118 %}
 7119 
 7120 // Store - reg, reg
 7121 // Eg. STR      x0, [sp, x1]
 7122 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7123 %{
 7124   single_instruction;
 7125   dst    : ISS(read);
 7126   src    : EX2(read);
 7127   INS01  : ISS;
 7128   LDST   : WR;
 7129 %}
 7130 
 7131 //------- Store pipeline operations -----------------------
 7132 
 7133 // Branch
 7134 pipe_class pipe_branch()
 7135 %{
 7136   single_instruction;
 7137   INS01  : ISS;
 7138   BRANCH : EX1;
 7139 %}
 7140 
 7141 // Conditional branch
 7142 pipe_class pipe_branch_cond(rFlagsReg cr)
 7143 %{
 7144   single_instruction;
 7145   cr     : EX1(read);
 7146   INS01  : ISS;
 7147   BRANCH : EX1;
 7148 %}
 7149 
 7150 // Compare & Branch
 7151 // EG.  CBZ/CBNZ
 7152 pipe_class pipe_cmp_branch(iRegI op1)
 7153 %{
 7154   single_instruction;
 7155   op1    : EX1(read);
 7156   INS01  : ISS;
 7157   BRANCH : EX1;
 7158 %}
 7159 
 7160 //------- Synchronisation operations ----------------------
 7161 
 7162 // Any operation requiring serialization.
 7163 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7164 pipe_class pipe_serial()
 7165 %{
 7166   single_instruction;
 7167   force_serialization;
 7168   fixed_latency(16);
 7169   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7170   LDST   : WR;
 7171 %}
 7172 
 7173 // Generic big/slow expanded idiom - also serialized
 7174 pipe_class pipe_slow()
 7175 %{
 7176   instruction_count(10);
 7177   multiple_bundles;
 7178   force_serialization;
 7179   fixed_latency(16);
 7180   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7181   LDST   : WR;
 7182 %}
 7183 
 7184 // Empty pipeline class
 7185 pipe_class pipe_class_empty()
 7186 %{
 7187   single_instruction;
 7188   fixed_latency(0);
 7189 %}
 7190 
 7191 // Default pipeline class.
 7192 pipe_class pipe_class_default()
 7193 %{
 7194   single_instruction;
 7195   fixed_latency(2);
 7196 %}
 7197 
 7198 // Pipeline class for compares.
 7199 pipe_class pipe_class_compare()
 7200 %{
 7201   single_instruction;
 7202   fixed_latency(16);
 7203 %}
 7204 
 7205 // Pipeline class for memory operations.
 7206 pipe_class pipe_class_memory()
 7207 %{
 7208   single_instruction;
 7209   fixed_latency(16);
 7210 %}
 7211 
 7212 // Pipeline class for call.
 7213 pipe_class pipe_class_call()
 7214 %{
 7215   single_instruction;
 7216   fixed_latency(100);
 7217 %}
 7218 
 7219 // Define the class for the Nop node.
 7220 define %{
 7221    MachNop = pipe_class_empty;
 7222 %}
 7223 
 7224 %}
 7225 //----------INSTRUCTIONS-------------------------------------------------------
 7226 //
 7227 // match      -- States which machine-independent subtree may be replaced
 7228 //               by this instruction.
 7229 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7230 //               selection to identify a minimum cost tree of machine
 7231 //               instructions that matches a tree of machine-independent
 7232 //               instructions.
 7233 // format     -- A string providing the disassembly for this instruction.
 7234 //               The value of an instruction's operand may be inserted
 7235 //               by referring to it with a '$' prefix.
 7236 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7237 //               to within an encode class as $primary, $secondary, and $tertiary
 7238 //               rrspectively.  The primary opcode is commonly used to
 7239 //               indicate the type of machine instruction, while secondary
 7240 //               and tertiary are often used for prefix options or addressing
 7241 //               modes.
 7242 // ins_encode -- A list of encode classes with parameters. The encode class
 7243 //               name must have been defined in an 'enc_class' specification
 7244 //               in the encode section of the architecture description.
 7245 
 7246 // ============================================================================
 7247 // Memory (Load/Store) Instructions
 7248 
 7249 // Load Instructions
 7250 
 7251 // Load Byte (8 bit signed)
 7252 instruct loadB(iRegINoSp dst, memory1 mem)
 7253 %{
 7254   match(Set dst (LoadB mem));
 7255   predicate(!needs_acquiring_load(n));
 7256 
 7257   ins_cost(4 * INSN_COST);
 7258   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7259 
 7260   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7261 
 7262   ins_pipe(iload_reg_mem);
 7263 %}
 7264 
 7265 // Load Byte (8 bit signed) into long
 7266 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7267 %{
 7268   match(Set dst (ConvI2L (LoadB mem)));
 7269   predicate(!needs_acquiring_load(n->in(1)));
 7270 
 7271   ins_cost(4 * INSN_COST);
 7272   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7273 
 7274   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7275 
 7276   ins_pipe(iload_reg_mem);
 7277 %}
 7278 
 7279 // Load Byte (8 bit unsigned)
 7280 instruct loadUB(iRegINoSp dst, memory1 mem)
 7281 %{
 7282   match(Set dst (LoadUB mem));
 7283   predicate(!needs_acquiring_load(n));
 7284 
 7285   ins_cost(4 * INSN_COST);
 7286   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7287 
 7288   ins_encode(aarch64_enc_ldrb(dst, mem));
 7289 
 7290   ins_pipe(iload_reg_mem);
 7291 %}
 7292 
 7293 // Load Byte (8 bit unsigned) into long
 7294 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7295 %{
 7296   match(Set dst (ConvI2L (LoadUB mem)));
 7297   predicate(!needs_acquiring_load(n->in(1)));
 7298 
 7299   ins_cost(4 * INSN_COST);
 7300   format %{ "ldrb  $dst, $mem\t# byte" %}
 7301 
 7302   ins_encode(aarch64_enc_ldrb(dst, mem));
 7303 
 7304   ins_pipe(iload_reg_mem);
 7305 %}
 7306 
 7307 // Load Short (16 bit signed)
 7308 instruct loadS(iRegINoSp dst, memory2 mem)
 7309 %{
 7310   match(Set dst (LoadS mem));
 7311   predicate(!needs_acquiring_load(n));
 7312 
 7313   ins_cost(4 * INSN_COST);
 7314   format %{ "ldrshw  $dst, $mem\t# short" %}
 7315 
 7316   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7317 
 7318   ins_pipe(iload_reg_mem);
 7319 %}
 7320 
 7321 // Load Short (16 bit signed) into long
 7322 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7323 %{
 7324   match(Set dst (ConvI2L (LoadS mem)));
 7325   predicate(!needs_acquiring_load(n->in(1)));
 7326 
 7327   ins_cost(4 * INSN_COST);
 7328   format %{ "ldrsh  $dst, $mem\t# short" %}
 7329 
 7330   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7331 
 7332   ins_pipe(iload_reg_mem);
 7333 %}
 7334 
 7335 // Load Char (16 bit unsigned)
 7336 instruct loadUS(iRegINoSp dst, memory2 mem)
 7337 %{
 7338   match(Set dst (LoadUS mem));
 7339   predicate(!needs_acquiring_load(n));
 7340 
 7341   ins_cost(4 * INSN_COST);
 7342   format %{ "ldrh  $dst, $mem\t# short" %}
 7343 
 7344   ins_encode(aarch64_enc_ldrh(dst, mem));
 7345 
 7346   ins_pipe(iload_reg_mem);
 7347 %}
 7348 
 7349 // Load Short/Char (16 bit unsigned) into long
 7350 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7351 %{
 7352   match(Set dst (ConvI2L (LoadUS mem)));
 7353   predicate(!needs_acquiring_load(n->in(1)));
 7354 
 7355   ins_cost(4 * INSN_COST);
 7356   format %{ "ldrh  $dst, $mem\t# short" %}
 7357 
 7358   ins_encode(aarch64_enc_ldrh(dst, mem));
 7359 
 7360   ins_pipe(iload_reg_mem);
 7361 %}
 7362 
 7363 // Load Integer (32 bit signed)
 7364 instruct loadI(iRegINoSp dst, memory4 mem)
 7365 %{
 7366   match(Set dst (LoadI mem));
 7367   predicate(!needs_acquiring_load(n));
 7368 
 7369   ins_cost(4 * INSN_COST);
 7370   format %{ "ldrw  $dst, $mem\t# int" %}
 7371 
 7372   ins_encode(aarch64_enc_ldrw(dst, mem));
 7373 
 7374   ins_pipe(iload_reg_mem);
 7375 %}
 7376 
 7377 // Load Integer (32 bit signed) into long
 7378 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7379 %{
 7380   match(Set dst (ConvI2L (LoadI mem)));
 7381   predicate(!needs_acquiring_load(n->in(1)));
 7382 
 7383   ins_cost(4 * INSN_COST);
 7384   format %{ "ldrsw  $dst, $mem\t# int" %}
 7385 
 7386   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7387 
 7388   ins_pipe(iload_reg_mem);
 7389 %}
 7390 
 7391 // Load Integer (32 bit unsigned) into long
 7392 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7393 %{
 7394   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7395   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7396 
 7397   ins_cost(4 * INSN_COST);
 7398   format %{ "ldrw  $dst, $mem\t# int" %}
 7399 
 7400   ins_encode(aarch64_enc_ldrw(dst, mem));
 7401 
 7402   ins_pipe(iload_reg_mem);
 7403 %}
 7404 
 7405 // Load Long (64 bit signed)
 7406 instruct loadL(iRegLNoSp dst, memory8 mem)
 7407 %{
 7408   match(Set dst (LoadL mem));
 7409   predicate(!needs_acquiring_load(n));
 7410 
 7411   ins_cost(4 * INSN_COST);
 7412   format %{ "ldr  $dst, $mem\t# int" %}
 7413 
 7414   ins_encode(aarch64_enc_ldr(dst, mem));
 7415 
 7416   ins_pipe(iload_reg_mem);
 7417 %}
 7418 
 7419 // Load Range
 7420 instruct loadRange(iRegINoSp dst, memory4 mem)
 7421 %{
 7422   match(Set dst (LoadRange mem));
 7423 
 7424   ins_cost(4 * INSN_COST);
 7425   format %{ "ldrw  $dst, $mem\t# range" %}
 7426 
 7427   ins_encode(aarch64_enc_ldrw(dst, mem));
 7428 
 7429   ins_pipe(iload_reg_mem);
 7430 %}
 7431 
 7432 // Load Pointer
 7433 instruct loadP(iRegPNoSp dst, memory8 mem)
 7434 %{
 7435   match(Set dst (LoadP mem));
 7436   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7437 
 7438   ins_cost(4 * INSN_COST);
 7439   format %{ "ldr  $dst, $mem\t# ptr" %}
 7440 
 7441   ins_encode(aarch64_enc_ldr(dst, mem));
 7442 
 7443   ins_pipe(iload_reg_mem);
 7444 %}
 7445 
 7446 // Load Compressed Pointer
 7447 instruct loadN(iRegNNoSp dst, memory4 mem)
 7448 %{
 7449   match(Set dst (LoadN mem));
 7450   predicate(!needs_acquiring_load(n));
 7451 
 7452   ins_cost(4 * INSN_COST);
 7453   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7454 
 7455   ins_encode(aarch64_enc_ldrw(dst, mem));
 7456 
 7457   ins_pipe(iload_reg_mem);
 7458 %}
 7459 
 7460 // Load Klass Pointer
 7461 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7462 %{
 7463   match(Set dst (LoadKlass mem));
 7464   predicate(!needs_acquiring_load(n));
 7465 
 7466   ins_cost(4 * INSN_COST);
 7467   format %{ "ldr  $dst, $mem\t# class" %}
 7468 
 7469   ins_encode(aarch64_enc_ldr(dst, mem));
 7470 
 7471   ins_pipe(iload_reg_mem);
 7472 %}
 7473 
 7474 // Load Narrow Klass Pointer
 7475 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7476 %{
 7477   match(Set dst (LoadNKlass mem));
 7478   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 7479 
 7480   ins_cost(4 * INSN_COST);
 7481   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7482 
 7483   ins_encode(aarch64_enc_ldrw(dst, mem));
 7484 
 7485   ins_pipe(iload_reg_mem);
 7486 %}
 7487 
 7488 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7489 %{
 7490   match(Set dst (LoadNKlass mem));
 7491   effect(KILL cr);
 7492   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 7493 
 7494   ins_cost(4 * INSN_COST);
 7495   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7496   ins_encode %{
 7497     __ load_nklass_compact($dst$$Register, $mem$$base$$Register, $mem$$index$$Register, $mem$$scale, $mem$$disp);
 7498   %}
 7499   ins_pipe(pipe_slow);
 7500 %}
 7501 
 7502 // Load Float
 7503 instruct loadF(vRegF dst, memory4 mem)
 7504 %{
 7505   match(Set dst (LoadF mem));
 7506   predicate(!needs_acquiring_load(n));
 7507 
 7508   ins_cost(4 * INSN_COST);
 7509   format %{ "ldrs  $dst, $mem\t# float" %}
 7510 
 7511   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7512 
 7513   ins_pipe(pipe_class_memory);
 7514 %}
 7515 
 7516 // Load Double
 7517 instruct loadD(vRegD dst, memory8 mem)
 7518 %{
 7519   match(Set dst (LoadD mem));
 7520   predicate(!needs_acquiring_load(n));
 7521 
 7522   ins_cost(4 * INSN_COST);
 7523   format %{ "ldrd  $dst, $mem\t# double" %}
 7524 
 7525   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7526 
 7527   ins_pipe(pipe_class_memory);
 7528 %}
 7529 
 7530 
 7531 // Load Int Constant
 7532 instruct loadConI(iRegINoSp dst, immI src)
 7533 %{
 7534   match(Set dst src);
 7535 
 7536   ins_cost(INSN_COST);
 7537   format %{ "mov $dst, $src\t# int" %}
 7538 
 7539   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7540 
 7541   ins_pipe(ialu_imm);
 7542 %}
 7543 
 7544 // Load Long Constant
 7545 instruct loadConL(iRegLNoSp dst, immL src)
 7546 %{
 7547   match(Set dst src);
 7548 
 7549   ins_cost(INSN_COST);
 7550   format %{ "mov $dst, $src\t# long" %}
 7551 
 7552   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7553 
 7554   ins_pipe(ialu_imm);
 7555 %}
 7556 
 7557 // Load Pointer Constant
 7558 
 7559 instruct loadConP(iRegPNoSp dst, immP con)
 7560 %{
 7561   match(Set dst con);
 7562 
 7563   ins_cost(INSN_COST * 4);
 7564   format %{
 7565     "mov  $dst, $con\t# ptr\n\t"
 7566   %}
 7567 
 7568   ins_encode(aarch64_enc_mov_p(dst, con));
 7569 
 7570   ins_pipe(ialu_imm);
 7571 %}
 7572 
 7573 // Load Null Pointer Constant
 7574 
 7575 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7576 %{
 7577   match(Set dst con);
 7578 
 7579   ins_cost(INSN_COST);
 7580   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7581 
 7582   ins_encode(aarch64_enc_mov_p0(dst, con));
 7583 
 7584   ins_pipe(ialu_imm);
 7585 %}
 7586 
 7587 // Load Pointer Constant One
 7588 
 7589 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7590 %{
 7591   match(Set dst con);
 7592 
 7593   ins_cost(INSN_COST);
 7594   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7595 
 7596   ins_encode(aarch64_enc_mov_p1(dst, con));
 7597 
 7598   ins_pipe(ialu_imm);
 7599 %}
 7600 
 7601 // Load Byte Map Base Constant
 7602 
 7603 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7604 %{
 7605   match(Set dst con);
 7606 
 7607   ins_cost(INSN_COST);
 7608   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7609 
 7610   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7611 
 7612   ins_pipe(ialu_imm);
 7613 %}
 7614 
 7615 // Load Narrow Pointer Constant
 7616 
 7617 instruct loadConN(iRegNNoSp dst, immN con)
 7618 %{
 7619   match(Set dst con);
 7620 
 7621   ins_cost(INSN_COST * 4);
 7622   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7623 
 7624   ins_encode(aarch64_enc_mov_n(dst, con));
 7625 
 7626   ins_pipe(ialu_imm);
 7627 %}
 7628 
 7629 // Load Narrow Null Pointer Constant
 7630 
 7631 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7632 %{
 7633   match(Set dst con);
 7634 
 7635   ins_cost(INSN_COST);
 7636   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7637 
 7638   ins_encode(aarch64_enc_mov_n0(dst, con));
 7639 
 7640   ins_pipe(ialu_imm);
 7641 %}
 7642 
 7643 // Load Narrow Klass Constant
 7644 
 7645 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7646 %{
 7647   match(Set dst con);
 7648 
 7649   ins_cost(INSN_COST);
 7650   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7651 
 7652   ins_encode(aarch64_enc_mov_nk(dst, con));
 7653 
 7654   ins_pipe(ialu_imm);
 7655 %}
 7656 
 7657 // Load Packed Float Constant
 7658 
 7659 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7660   match(Set dst con);
 7661   ins_cost(INSN_COST * 4);
 7662   format %{ "fmovs  $dst, $con"%}
 7663   ins_encode %{
 7664     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7665   %}
 7666 
 7667   ins_pipe(fp_imm_s);
 7668 %}
 7669 
 7670 // Load Float Constant
 7671 
 7672 instruct loadConF(vRegF dst, immF con) %{
 7673   match(Set dst con);
 7674 
 7675   ins_cost(INSN_COST * 4);
 7676 
 7677   format %{
 7678     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7679   %}
 7680 
 7681   ins_encode %{
 7682     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7683   %}
 7684 
 7685   ins_pipe(fp_load_constant_s);
 7686 %}
 7687 
 7688 // Load Packed Double Constant
 7689 
 7690 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7691   match(Set dst con);
 7692   ins_cost(INSN_COST);
 7693   format %{ "fmovd  $dst, $con"%}
 7694   ins_encode %{
 7695     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7696   %}
 7697 
 7698   ins_pipe(fp_imm_d);
 7699 %}
 7700 
 7701 // Load Double Constant
 7702 
 7703 instruct loadConD(vRegD dst, immD con) %{
 7704   match(Set dst con);
 7705 
 7706   ins_cost(INSN_COST * 5);
 7707   format %{
 7708     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7709   %}
 7710 
 7711   ins_encode %{
 7712     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7713   %}
 7714 
 7715   ins_pipe(fp_load_constant_d);
 7716 %}
 7717 
 7718 // Store Instructions
 7719 
 7720 // Store CMS card-mark Immediate
 7721 instruct storeimmCM0(immI0 zero, memory1 mem)
 7722 %{
 7723   match(Set mem (StoreCM mem zero));
 7724 
 7725   ins_cost(INSN_COST);
 7726   format %{ "storestore (elided)\n\t"
 7727             "strb zr, $mem\t# byte" %}
 7728 
 7729   ins_encode(aarch64_enc_strb0(mem));
 7730 
 7731   ins_pipe(istore_mem);
 7732 %}
 7733 
 7734 // Store CMS card-mark Immediate with intervening StoreStore
 7735 // needed when using CMS with no conditional card marking
 7736 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7737 %{
 7738   match(Set mem (StoreCM mem zero));
 7739 
 7740   ins_cost(INSN_COST * 2);
 7741   format %{ "storestore\n\t"
 7742             "dmb ishst"
 7743             "\n\tstrb zr, $mem\t# byte" %}
 7744 
 7745   ins_encode(aarch64_enc_strb0_ordered(mem));
 7746 
 7747   ins_pipe(istore_mem);
 7748 %}
 7749 
 7750 // Store Byte
 7751 instruct storeB(iRegIorL2I src, memory1 mem)
 7752 %{
 7753   match(Set mem (StoreB mem src));
 7754   predicate(!needs_releasing_store(n));
 7755 
 7756   ins_cost(INSN_COST);
 7757   format %{ "strb  $src, $mem\t# byte" %}
 7758 
 7759   ins_encode(aarch64_enc_strb(src, mem));
 7760 
 7761   ins_pipe(istore_reg_mem);
 7762 %}
 7763 
 7764 
 7765 instruct storeimmB0(immI0 zero, memory1 mem)
 7766 %{
 7767   match(Set mem (StoreB mem zero));
 7768   predicate(!needs_releasing_store(n));
 7769 
 7770   ins_cost(INSN_COST);
 7771   format %{ "strb rscractch2, $mem\t# byte" %}
 7772 
 7773   ins_encode(aarch64_enc_strb0(mem));
 7774 
 7775   ins_pipe(istore_mem);
 7776 %}
 7777 
 7778 // Store Char/Short
 7779 instruct storeC(iRegIorL2I src, memory2 mem)
 7780 %{
 7781   match(Set mem (StoreC mem src));
 7782   predicate(!needs_releasing_store(n));
 7783 
 7784   ins_cost(INSN_COST);
 7785   format %{ "strh  $src, $mem\t# short" %}
 7786 
 7787   ins_encode(aarch64_enc_strh(src, mem));
 7788 
 7789   ins_pipe(istore_reg_mem);
 7790 %}
 7791 
 7792 instruct storeimmC0(immI0 zero, memory2 mem)
 7793 %{
 7794   match(Set mem (StoreC mem zero));
 7795   predicate(!needs_releasing_store(n));
 7796 
 7797   ins_cost(INSN_COST);
 7798   format %{ "strh  zr, $mem\t# short" %}
 7799 
 7800   ins_encode(aarch64_enc_strh0(mem));
 7801 
 7802   ins_pipe(istore_mem);
 7803 %}
 7804 
 7805 // Store Integer
 7806 
 7807 instruct storeI(iRegIorL2I src, memory4 mem)
 7808 %{
 7809   match(Set mem(StoreI mem src));
 7810   predicate(!needs_releasing_store(n));
 7811 
 7812   ins_cost(INSN_COST);
 7813   format %{ "strw  $src, $mem\t# int" %}
 7814 
 7815   ins_encode(aarch64_enc_strw(src, mem));
 7816 
 7817   ins_pipe(istore_reg_mem);
 7818 %}
 7819 
 7820 instruct storeimmI0(immI0 zero, memory4 mem)
 7821 %{
 7822   match(Set mem(StoreI mem zero));
 7823   predicate(!needs_releasing_store(n));
 7824 
 7825   ins_cost(INSN_COST);
 7826   format %{ "strw  zr, $mem\t# int" %}
 7827 
 7828   ins_encode(aarch64_enc_strw0(mem));
 7829 
 7830   ins_pipe(istore_mem);
 7831 %}
 7832 
 7833 // Store Long (64 bit signed)
 7834 instruct storeL(iRegL src, memory8 mem)
 7835 %{
 7836   match(Set mem (StoreL mem src));
 7837   predicate(!needs_releasing_store(n));
 7838 
 7839   ins_cost(INSN_COST);
 7840   format %{ "str  $src, $mem\t# int" %}
 7841 
 7842   ins_encode(aarch64_enc_str(src, mem));
 7843 
 7844   ins_pipe(istore_reg_mem);
 7845 %}
 7846 
 7847 // Store Long (64 bit signed)
 7848 instruct storeimmL0(immL0 zero, memory8 mem)
 7849 %{
 7850   match(Set mem (StoreL mem zero));
 7851   predicate(!needs_releasing_store(n));
 7852 
 7853   ins_cost(INSN_COST);
 7854   format %{ "str  zr, $mem\t# int" %}
 7855 
 7856   ins_encode(aarch64_enc_str0(mem));
 7857 
 7858   ins_pipe(istore_mem);
 7859 %}
 7860 
 7861 // Store Pointer
 7862 instruct storeP(iRegP src, memory8 mem)
 7863 %{
 7864   match(Set mem (StoreP mem src));
 7865   predicate(!needs_releasing_store(n));
 7866 
 7867   ins_cost(INSN_COST);
 7868   format %{ "str  $src, $mem\t# ptr" %}
 7869 
 7870   ins_encode(aarch64_enc_str(src, mem));
 7871 
 7872   ins_pipe(istore_reg_mem);
 7873 %}
 7874 
 7875 // Store Pointer
 7876 instruct storeimmP0(immP0 zero, memory8 mem)
 7877 %{
 7878   match(Set mem (StoreP mem zero));
 7879   predicate(!needs_releasing_store(n));
 7880 
 7881   ins_cost(INSN_COST);
 7882   format %{ "str zr, $mem\t# ptr" %}
 7883 
 7884   ins_encode(aarch64_enc_str0(mem));
 7885 
 7886   ins_pipe(istore_mem);
 7887 %}
 7888 
 7889 // Store Compressed Pointer
 7890 instruct storeN(iRegN src, memory4 mem)
 7891 %{
 7892   match(Set mem (StoreN mem src));
 7893   predicate(!needs_releasing_store(n));
 7894 
 7895   ins_cost(INSN_COST);
 7896   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7897 
 7898   ins_encode(aarch64_enc_strw(src, mem));
 7899 
 7900   ins_pipe(istore_reg_mem);
 7901 %}
 7902 
 7903 instruct storeImmN0(immN0 zero, memory4 mem)
 7904 %{
 7905   match(Set mem (StoreN mem zero));
 7906   predicate(!needs_releasing_store(n));
 7907 
 7908   ins_cost(INSN_COST);
 7909   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7910 
 7911   ins_encode(aarch64_enc_strw0(mem));
 7912 
 7913   ins_pipe(istore_mem);
 7914 %}
 7915 
 7916 // Store Float
 7917 instruct storeF(vRegF src, memory4 mem)
 7918 %{
 7919   match(Set mem (StoreF mem src));
 7920   predicate(!needs_releasing_store(n));
 7921 
 7922   ins_cost(INSN_COST);
 7923   format %{ "strs  $src, $mem\t# float" %}
 7924 
 7925   ins_encode( aarch64_enc_strs(src, mem) );
 7926 
 7927   ins_pipe(pipe_class_memory);
 7928 %}
 7929 
 7930 // TODO
 7931 // implement storeImmF0 and storeFImmPacked
 7932 
 7933 // Store Double
 7934 instruct storeD(vRegD src, memory8 mem)
 7935 %{
 7936   match(Set mem (StoreD mem src));
 7937   predicate(!needs_releasing_store(n));
 7938 
 7939   ins_cost(INSN_COST);
 7940   format %{ "strd  $src, $mem\t# double" %}
 7941 
 7942   ins_encode( aarch64_enc_strd(src, mem) );
 7943 
 7944   ins_pipe(pipe_class_memory);
 7945 %}
 7946 
 7947 // Store Compressed Klass Pointer
 7948 instruct storeNKlass(iRegN src, memory4 mem)
 7949 %{
 7950   predicate(!needs_releasing_store(n));
 7951   match(Set mem (StoreNKlass mem src));
 7952 
 7953   ins_cost(INSN_COST);
 7954   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7955 
 7956   ins_encode(aarch64_enc_strw(src, mem));
 7957 
 7958   ins_pipe(istore_reg_mem);
 7959 %}
 7960 
 7961 // TODO
 7962 // implement storeImmD0 and storeDImmPacked
 7963 
 7964 // prefetch instructions
 7965 // Must be safe to execute with invalid address (cannot fault).
 7966 
 7967 instruct prefetchalloc( memory8 mem ) %{
 7968   match(PrefetchAllocation mem);
 7969 
 7970   ins_cost(INSN_COST);
 7971   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7972 
 7973   ins_encode( aarch64_enc_prefetchw(mem) );
 7974 
 7975   ins_pipe(iload_prefetch);
 7976 %}
 7977 
 7978 //  ---------------- volatile loads and stores ----------------
 7979 
 7980 // Load Byte (8 bit signed)
 7981 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7982 %{
 7983   match(Set dst (LoadB mem));
 7984 
 7985   ins_cost(VOLATILE_REF_COST);
 7986   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7987 
 7988   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7989 
 7990   ins_pipe(pipe_serial);
 7991 %}
 7992 
 7993 // Load Byte (8 bit signed) into long
 7994 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7995 %{
 7996   match(Set dst (ConvI2L (LoadB mem)));
 7997 
 7998   ins_cost(VOLATILE_REF_COST);
 7999   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8000 
 8001   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8002 
 8003   ins_pipe(pipe_serial);
 8004 %}
 8005 
 8006 // Load Byte (8 bit unsigned)
 8007 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8008 %{
 8009   match(Set dst (LoadUB mem));
 8010 
 8011   ins_cost(VOLATILE_REF_COST);
 8012   format %{ "ldarb  $dst, $mem\t# byte" %}
 8013 
 8014   ins_encode(aarch64_enc_ldarb(dst, mem));
 8015 
 8016   ins_pipe(pipe_serial);
 8017 %}
 8018 
 8019 // Load Byte (8 bit unsigned) into long
 8020 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8021 %{
 8022   match(Set dst (ConvI2L (LoadUB mem)));
 8023 
 8024   ins_cost(VOLATILE_REF_COST);
 8025   format %{ "ldarb  $dst, $mem\t# byte" %}
 8026 
 8027   ins_encode(aarch64_enc_ldarb(dst, mem));
 8028 
 8029   ins_pipe(pipe_serial);
 8030 %}
 8031 
 8032 // Load Short (16 bit signed)
 8033 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8034 %{
 8035   match(Set dst (LoadS mem));
 8036 
 8037   ins_cost(VOLATILE_REF_COST);
 8038   format %{ "ldarshw  $dst, $mem\t# short" %}
 8039 
 8040   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8041 
 8042   ins_pipe(pipe_serial);
 8043 %}
 8044 
 8045 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8046 %{
 8047   match(Set dst (LoadUS mem));
 8048 
 8049   ins_cost(VOLATILE_REF_COST);
 8050   format %{ "ldarhw  $dst, $mem\t# short" %}
 8051 
 8052   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8053 
 8054   ins_pipe(pipe_serial);
 8055 %}
 8056 
 8057 // Load Short/Char (16 bit unsigned) into long
 8058 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8059 %{
 8060   match(Set dst (ConvI2L (LoadUS mem)));
 8061 
 8062   ins_cost(VOLATILE_REF_COST);
 8063   format %{ "ldarh  $dst, $mem\t# short" %}
 8064 
 8065   ins_encode(aarch64_enc_ldarh(dst, mem));
 8066 
 8067   ins_pipe(pipe_serial);
 8068 %}
 8069 
 8070 // Load Short/Char (16 bit signed) into long
 8071 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8072 %{
 8073   match(Set dst (ConvI2L (LoadS mem)));
 8074 
 8075   ins_cost(VOLATILE_REF_COST);
 8076   format %{ "ldarh  $dst, $mem\t# short" %}
 8077 
 8078   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8079 
 8080   ins_pipe(pipe_serial);
 8081 %}
 8082 
 8083 // Load Integer (32 bit signed)
 8084 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8085 %{
 8086   match(Set dst (LoadI mem));
 8087 
 8088   ins_cost(VOLATILE_REF_COST);
 8089   format %{ "ldarw  $dst, $mem\t# int" %}
 8090 
 8091   ins_encode(aarch64_enc_ldarw(dst, mem));
 8092 
 8093   ins_pipe(pipe_serial);
 8094 %}
 8095 
 8096 // Load Integer (32 bit unsigned) into long
 8097 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8098 %{
 8099   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8100 
 8101   ins_cost(VOLATILE_REF_COST);
 8102   format %{ "ldarw  $dst, $mem\t# int" %}
 8103 
 8104   ins_encode(aarch64_enc_ldarw(dst, mem));
 8105 
 8106   ins_pipe(pipe_serial);
 8107 %}
 8108 
 8109 // Load Long (64 bit signed)
 8110 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8111 %{
 8112   match(Set dst (LoadL mem));
 8113 
 8114   ins_cost(VOLATILE_REF_COST);
 8115   format %{ "ldar  $dst, $mem\t# int" %}
 8116 
 8117   ins_encode(aarch64_enc_ldar(dst, mem));
 8118 
 8119   ins_pipe(pipe_serial);
 8120 %}
 8121 
 8122 // Load Pointer
 8123 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8124 %{
 8125   match(Set dst (LoadP mem));
 8126   predicate(n->as_Load()->barrier_data() == 0);
 8127 
 8128   ins_cost(VOLATILE_REF_COST);
 8129   format %{ "ldar  $dst, $mem\t# ptr" %}
 8130 
 8131   ins_encode(aarch64_enc_ldar(dst, mem));
 8132 
 8133   ins_pipe(pipe_serial);
 8134 %}
 8135 
 8136 // Load Compressed Pointer
 8137 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8138 %{
 8139   match(Set dst (LoadN mem));
 8140 
 8141   ins_cost(VOLATILE_REF_COST);
 8142   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8143 
 8144   ins_encode(aarch64_enc_ldarw(dst, mem));
 8145 
 8146   ins_pipe(pipe_serial);
 8147 %}
 8148 
 8149 // Load Float
 8150 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8151 %{
 8152   match(Set dst (LoadF mem));
 8153 
 8154   ins_cost(VOLATILE_REF_COST);
 8155   format %{ "ldars  $dst, $mem\t# float" %}
 8156 
 8157   ins_encode( aarch64_enc_fldars(dst, mem) );
 8158 
 8159   ins_pipe(pipe_serial);
 8160 %}
 8161 
 8162 // Load Double
 8163 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8164 %{
 8165   match(Set dst (LoadD mem));
 8166 
 8167   ins_cost(VOLATILE_REF_COST);
 8168   format %{ "ldard  $dst, $mem\t# double" %}
 8169 
 8170   ins_encode( aarch64_enc_fldard(dst, mem) );
 8171 
 8172   ins_pipe(pipe_serial);
 8173 %}
 8174 
 8175 // Store Byte
 8176 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8177 %{
 8178   match(Set mem (StoreB mem src));
 8179 
 8180   ins_cost(VOLATILE_REF_COST);
 8181   format %{ "stlrb  $src, $mem\t# byte" %}
 8182 
 8183   ins_encode(aarch64_enc_stlrb(src, mem));
 8184 
 8185   ins_pipe(pipe_class_memory);
 8186 %}
 8187 
 8188 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8189 %{
 8190   match(Set mem (StoreB mem zero));
 8191 
 8192   ins_cost(VOLATILE_REF_COST);
 8193   format %{ "stlrb  zr, $mem\t# byte" %}
 8194 
 8195   ins_encode(aarch64_enc_stlrb0(mem));
 8196 
 8197   ins_pipe(pipe_class_memory);
 8198 %}
 8199 
 8200 // Store Char/Short
 8201 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8202 %{
 8203   match(Set mem (StoreC mem src));
 8204 
 8205   ins_cost(VOLATILE_REF_COST);
 8206   format %{ "stlrh  $src, $mem\t# short" %}
 8207 
 8208   ins_encode(aarch64_enc_stlrh(src, mem));
 8209 
 8210   ins_pipe(pipe_class_memory);
 8211 %}
 8212 
 8213 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8214 %{
 8215   match(Set mem (StoreC mem zero));
 8216 
 8217   ins_cost(VOLATILE_REF_COST);
 8218   format %{ "stlrh  zr, $mem\t# short" %}
 8219 
 8220   ins_encode(aarch64_enc_stlrh0(mem));
 8221 
 8222   ins_pipe(pipe_class_memory);
 8223 %}
 8224 
 8225 // Store Integer
 8226 
 8227 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8228 %{
 8229   match(Set mem(StoreI mem src));
 8230 
 8231   ins_cost(VOLATILE_REF_COST);
 8232   format %{ "stlrw  $src, $mem\t# int" %}
 8233 
 8234   ins_encode(aarch64_enc_stlrw(src, mem));
 8235 
 8236   ins_pipe(pipe_class_memory);
 8237 %}
 8238 
 8239 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8240 %{
 8241   match(Set mem(StoreI mem zero));
 8242 
 8243   ins_cost(VOLATILE_REF_COST);
 8244   format %{ "stlrw  zr, $mem\t# int" %}
 8245 
 8246   ins_encode(aarch64_enc_stlrw0(mem));
 8247 
 8248   ins_pipe(pipe_class_memory);
 8249 %}
 8250 
 8251 // Store Long (64 bit signed)
 8252 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8253 %{
 8254   match(Set mem (StoreL mem src));
 8255 
 8256   ins_cost(VOLATILE_REF_COST);
 8257   format %{ "stlr  $src, $mem\t# int" %}
 8258 
 8259   ins_encode(aarch64_enc_stlr(src, mem));
 8260 
 8261   ins_pipe(pipe_class_memory);
 8262 %}
 8263 
 8264 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8265 %{
 8266   match(Set mem (StoreL mem zero));
 8267 
 8268   ins_cost(VOLATILE_REF_COST);
 8269   format %{ "stlr  zr, $mem\t# int" %}
 8270 
 8271   ins_encode(aarch64_enc_stlr0(mem));
 8272 
 8273   ins_pipe(pipe_class_memory);
 8274 %}
 8275 
 8276 // Store Pointer
 8277 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8278 %{
 8279   match(Set mem (StoreP mem src));
 8280 
 8281   ins_cost(VOLATILE_REF_COST);
 8282   format %{ "stlr  $src, $mem\t# ptr" %}
 8283 
 8284   ins_encode(aarch64_enc_stlr(src, mem));
 8285 
 8286   ins_pipe(pipe_class_memory);
 8287 %}
 8288 
 8289 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8290 %{
 8291   match(Set mem (StoreP mem zero));
 8292 
 8293   ins_cost(VOLATILE_REF_COST);
 8294   format %{ "stlr  zr, $mem\t# ptr" %}
 8295 
 8296   ins_encode(aarch64_enc_stlr0(mem));
 8297 
 8298   ins_pipe(pipe_class_memory);
 8299 %}
 8300 
 8301 // Store Compressed Pointer
 8302 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8303 %{
 8304   match(Set mem (StoreN mem src));
 8305 
 8306   ins_cost(VOLATILE_REF_COST);
 8307   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8308 
 8309   ins_encode(aarch64_enc_stlrw(src, mem));
 8310 
 8311   ins_pipe(pipe_class_memory);
 8312 %}
 8313 
 8314 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8315 %{
 8316   match(Set mem (StoreN mem zero));
 8317 
 8318   ins_cost(VOLATILE_REF_COST);
 8319   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8320 
 8321   ins_encode(aarch64_enc_stlrw0(mem));
 8322 
 8323   ins_pipe(pipe_class_memory);
 8324 %}
 8325 
 8326 // Store Float
 8327 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8328 %{
 8329   match(Set mem (StoreF mem src));
 8330 
 8331   ins_cost(VOLATILE_REF_COST);
 8332   format %{ "stlrs  $src, $mem\t# float" %}
 8333 
 8334   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8335 
 8336   ins_pipe(pipe_class_memory);
 8337 %}
 8338 
 8339 // TODO
 8340 // implement storeImmF0 and storeFImmPacked
 8341 
 8342 // Store Double
 8343 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8344 %{
 8345   match(Set mem (StoreD mem src));
 8346 
 8347   ins_cost(VOLATILE_REF_COST);
 8348   format %{ "stlrd  $src, $mem\t# double" %}
 8349 
 8350   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8351 
 8352   ins_pipe(pipe_class_memory);
 8353 %}
 8354 
 8355 //  ---------------- end of volatile loads and stores ----------------
 8356 
 8357 instruct cacheWB(indirect addr)
 8358 %{
 8359   predicate(VM_Version::supports_data_cache_line_flush());
 8360   match(CacheWB addr);
 8361 
 8362   ins_cost(100);
 8363   format %{"cache wb $addr" %}
 8364   ins_encode %{
 8365     assert($addr->index_position() < 0, "should be");
 8366     assert($addr$$disp == 0, "should be");
 8367     __ cache_wb(Address($addr$$base$$Register, 0));
 8368   %}
 8369   ins_pipe(pipe_slow); // XXX
 8370 %}
 8371 
 8372 instruct cacheWBPreSync()
 8373 %{
 8374   predicate(VM_Version::supports_data_cache_line_flush());
 8375   match(CacheWBPreSync);
 8376 
 8377   ins_cost(100);
 8378   format %{"cache wb presync" %}
 8379   ins_encode %{
 8380     __ cache_wbsync(true);
 8381   %}
 8382   ins_pipe(pipe_slow); // XXX
 8383 %}
 8384 
 8385 instruct cacheWBPostSync()
 8386 %{
 8387   predicate(VM_Version::supports_data_cache_line_flush());
 8388   match(CacheWBPostSync);
 8389 
 8390   ins_cost(100);
 8391   format %{"cache wb postsync" %}
 8392   ins_encode %{
 8393     __ cache_wbsync(false);
 8394   %}
 8395   ins_pipe(pipe_slow); // XXX
 8396 %}
 8397 
 8398 // ============================================================================
 8399 // BSWAP Instructions
 8400 
 8401 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8402   match(Set dst (ReverseBytesI src));
 8403 
 8404   ins_cost(INSN_COST);
 8405   format %{ "revw  $dst, $src" %}
 8406 
 8407   ins_encode %{
 8408     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8409   %}
 8410 
 8411   ins_pipe(ialu_reg);
 8412 %}
 8413 
 8414 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8415   match(Set dst (ReverseBytesL src));
 8416 
 8417   ins_cost(INSN_COST);
 8418   format %{ "rev  $dst, $src" %}
 8419 
 8420   ins_encode %{
 8421     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8422   %}
 8423 
 8424   ins_pipe(ialu_reg);
 8425 %}
 8426 
 8427 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8428   match(Set dst (ReverseBytesUS src));
 8429 
 8430   ins_cost(INSN_COST);
 8431   format %{ "rev16w  $dst, $src" %}
 8432 
 8433   ins_encode %{
 8434     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8435   %}
 8436 
 8437   ins_pipe(ialu_reg);
 8438 %}
 8439 
 8440 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8441   match(Set dst (ReverseBytesS src));
 8442 
 8443   ins_cost(INSN_COST);
 8444   format %{ "rev16w  $dst, $src\n\t"
 8445             "sbfmw $dst, $dst, #0, #15" %}
 8446 
 8447   ins_encode %{
 8448     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8449     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8450   %}
 8451 
 8452   ins_pipe(ialu_reg);
 8453 %}
 8454 
 8455 // ============================================================================
 8456 // Zero Count Instructions
 8457 
 8458 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8459   match(Set dst (CountLeadingZerosI src));
 8460 
 8461   ins_cost(INSN_COST);
 8462   format %{ "clzw  $dst, $src" %}
 8463   ins_encode %{
 8464     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8465   %}
 8466 
 8467   ins_pipe(ialu_reg);
 8468 %}
 8469 
 8470 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8471   match(Set dst (CountLeadingZerosL src));
 8472 
 8473   ins_cost(INSN_COST);
 8474   format %{ "clz   $dst, $src" %}
 8475   ins_encode %{
 8476     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8477   %}
 8478 
 8479   ins_pipe(ialu_reg);
 8480 %}
 8481 
 8482 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8483   match(Set dst (CountTrailingZerosI src));
 8484 
 8485   ins_cost(INSN_COST * 2);
 8486   format %{ "rbitw  $dst, $src\n\t"
 8487             "clzw   $dst, $dst" %}
 8488   ins_encode %{
 8489     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8490     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8491   %}
 8492 
 8493   ins_pipe(ialu_reg);
 8494 %}
 8495 
 8496 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8497   match(Set dst (CountTrailingZerosL src));
 8498 
 8499   ins_cost(INSN_COST * 2);
 8500   format %{ "rbit   $dst, $src\n\t"
 8501             "clz    $dst, $dst" %}
 8502   ins_encode %{
 8503     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8504     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8505   %}
 8506 
 8507   ins_pipe(ialu_reg);
 8508 %}
 8509 
 8510 //---------- Population Count Instructions -------------------------------------
 8511 //
 8512 
 8513 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8514   predicate(UsePopCountInstruction);
 8515   match(Set dst (PopCountI src));
 8516   effect(TEMP tmp);
 8517   ins_cost(INSN_COST * 13);
 8518 
 8519   format %{ "movw   $src, $src\n\t"
 8520             "mov    $tmp, $src\t# vector (1D)\n\t"
 8521             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8522             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8523             "mov    $dst, $tmp\t# vector (1D)" %}
 8524   ins_encode %{
 8525     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8526     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8527     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8528     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8529     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8530   %}
 8531 
 8532   ins_pipe(pipe_class_default);
 8533 %}
 8534 
 8535 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8536   predicate(UsePopCountInstruction);
 8537   match(Set dst (PopCountI (LoadI mem)));
 8538   effect(TEMP tmp);
 8539   ins_cost(INSN_COST * 13);
 8540 
 8541   format %{ "ldrs   $tmp, $mem\n\t"
 8542             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8543             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8544             "mov    $dst, $tmp\t# vector (1D)" %}
 8545   ins_encode %{
 8546     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8547     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8548               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8549     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8550     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8551     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8552   %}
 8553 
 8554   ins_pipe(pipe_class_default);
 8555 %}
 8556 
 8557 // Note: Long.bitCount(long) returns an int.
 8558 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8559   predicate(UsePopCountInstruction);
 8560   match(Set dst (PopCountL src));
 8561   effect(TEMP tmp);
 8562   ins_cost(INSN_COST * 13);
 8563 
 8564   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8565             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8566             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8567             "mov    $dst, $tmp\t# vector (1D)" %}
 8568   ins_encode %{
 8569     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8570     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8571     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8572     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8573   %}
 8574 
 8575   ins_pipe(pipe_class_default);
 8576 %}
 8577 
 8578 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8579   predicate(UsePopCountInstruction);
 8580   match(Set dst (PopCountL (LoadL mem)));
 8581   effect(TEMP tmp);
 8582   ins_cost(INSN_COST * 13);
 8583 
 8584   format %{ "ldrd   $tmp, $mem\n\t"
 8585             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8586             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8587             "mov    $dst, $tmp\t# vector (1D)" %}
 8588   ins_encode %{
 8589     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8590     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8591               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8592     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8593     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8594     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8595   %}
 8596 
 8597   ins_pipe(pipe_class_default);
 8598 %}
 8599 
 8600 // ============================================================================
 8601 // MemBar Instruction
 8602 
 8603 instruct load_fence() %{
 8604   match(LoadFence);
 8605   ins_cost(VOLATILE_REF_COST);
 8606 
 8607   format %{ "load_fence" %}
 8608 
 8609   ins_encode %{
 8610     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8611   %}
 8612   ins_pipe(pipe_serial);
 8613 %}
 8614 
 8615 instruct unnecessary_membar_acquire() %{
 8616   predicate(unnecessary_acquire(n));
 8617   match(MemBarAcquire);
 8618   ins_cost(0);
 8619 
 8620   format %{ "membar_acquire (elided)" %}
 8621 
 8622   ins_encode %{
 8623     __ block_comment("membar_acquire (elided)");
 8624   %}
 8625 
 8626   ins_pipe(pipe_class_empty);
 8627 %}
 8628 
 8629 instruct membar_acquire() %{
 8630   match(MemBarAcquire);
 8631   ins_cost(VOLATILE_REF_COST);
 8632 
 8633   format %{ "membar_acquire\n\t"
 8634             "dmb ish" %}
 8635 
 8636   ins_encode %{
 8637     __ block_comment("membar_acquire");
 8638     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8639   %}
 8640 
 8641   ins_pipe(pipe_serial);
 8642 %}
 8643 
 8644 
 8645 instruct membar_acquire_lock() %{
 8646   match(MemBarAcquireLock);
 8647   ins_cost(VOLATILE_REF_COST);
 8648 
 8649   format %{ "membar_acquire_lock (elided)" %}
 8650 
 8651   ins_encode %{
 8652     __ block_comment("membar_acquire_lock (elided)");
 8653   %}
 8654 
 8655   ins_pipe(pipe_serial);
 8656 %}
 8657 
 8658 instruct store_fence() %{
 8659   match(StoreFence);
 8660   ins_cost(VOLATILE_REF_COST);
 8661 
 8662   format %{ "store_fence" %}
 8663 
 8664   ins_encode %{
 8665     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8666   %}
 8667   ins_pipe(pipe_serial);
 8668 %}
 8669 
 8670 instruct unnecessary_membar_release() %{
 8671   predicate(unnecessary_release(n));
 8672   match(MemBarRelease);
 8673   ins_cost(0);
 8674 
 8675   format %{ "membar_release (elided)" %}
 8676 
 8677   ins_encode %{
 8678     __ block_comment("membar_release (elided)");
 8679   %}
 8680   ins_pipe(pipe_serial);
 8681 %}
 8682 
 8683 instruct membar_release() %{
 8684   match(MemBarRelease);
 8685   ins_cost(VOLATILE_REF_COST);
 8686 
 8687   format %{ "membar_release\n\t"
 8688             "dmb ish" %}
 8689 
 8690   ins_encode %{
 8691     __ block_comment("membar_release");
 8692     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8693   %}
 8694   ins_pipe(pipe_serial);
 8695 %}
 8696 
 8697 instruct membar_storestore() %{
 8698   match(MemBarStoreStore);
 8699   match(StoreStoreFence);
 8700   ins_cost(VOLATILE_REF_COST);
 8701 
 8702   format %{ "MEMBAR-store-store" %}
 8703 
 8704   ins_encode %{
 8705     __ membar(Assembler::StoreStore);
 8706   %}
 8707   ins_pipe(pipe_serial);
 8708 %}
 8709 
 8710 instruct membar_release_lock() %{
 8711   match(MemBarReleaseLock);
 8712   ins_cost(VOLATILE_REF_COST);
 8713 
 8714   format %{ "membar_release_lock (elided)" %}
 8715 
 8716   ins_encode %{
 8717     __ block_comment("membar_release_lock (elided)");
 8718   %}
 8719 
 8720   ins_pipe(pipe_serial);
 8721 %}
 8722 
 8723 instruct unnecessary_membar_volatile() %{
 8724   predicate(unnecessary_volatile(n));
 8725   match(MemBarVolatile);
 8726   ins_cost(0);
 8727 
 8728   format %{ "membar_volatile (elided)" %}
 8729 
 8730   ins_encode %{
 8731     __ block_comment("membar_volatile (elided)");
 8732   %}
 8733 
 8734   ins_pipe(pipe_serial);
 8735 %}
 8736 
 8737 instruct membar_volatile() %{
 8738   match(MemBarVolatile);
 8739   ins_cost(VOLATILE_REF_COST*100);
 8740 
 8741   format %{ "membar_volatile\n\t"
 8742              "dmb ish"%}
 8743 
 8744   ins_encode %{
 8745     __ block_comment("membar_volatile");
 8746     __ membar(Assembler::StoreLoad);
 8747   %}
 8748 
 8749   ins_pipe(pipe_serial);
 8750 %}
 8751 
 8752 // ============================================================================
 8753 // Cast/Convert Instructions
 8754 
 8755 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8756   match(Set dst (CastX2P src));
 8757 
 8758   ins_cost(INSN_COST);
 8759   format %{ "mov $dst, $src\t# long -> ptr" %}
 8760 
 8761   ins_encode %{
 8762     if ($dst$$reg != $src$$reg) {
 8763       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8764     }
 8765   %}
 8766 
 8767   ins_pipe(ialu_reg);
 8768 %}
 8769 
 8770 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8771   match(Set dst (CastP2X src));
 8772 
 8773   ins_cost(INSN_COST);
 8774   format %{ "mov $dst, $src\t# ptr -> long" %}
 8775 
 8776   ins_encode %{
 8777     if ($dst$$reg != $src$$reg) {
 8778       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8779     }
 8780   %}
 8781 
 8782   ins_pipe(ialu_reg);
 8783 %}
 8784 
 8785 // Convert oop into int for vectors alignment masking
 8786 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8787   match(Set dst (ConvL2I (CastP2X src)));
 8788 
 8789   ins_cost(INSN_COST);
 8790   format %{ "movw $dst, $src\t# ptr -> int" %}
 8791   ins_encode %{
 8792     __ movw($dst$$Register, $src$$Register);
 8793   %}
 8794 
 8795   ins_pipe(ialu_reg);
 8796 %}
 8797 
 8798 // Convert compressed oop into int for vectors alignment masking
 8799 // in case of 32bit oops (heap < 4Gb).
 8800 instruct convN2I(iRegINoSp dst, iRegN src)
 8801 %{
 8802   predicate(CompressedOops::shift() == 0);
 8803   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8804 
 8805   ins_cost(INSN_COST);
 8806   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8807   ins_encode %{
 8808     __ movw($dst$$Register, $src$$Register);
 8809   %}
 8810 
 8811   ins_pipe(ialu_reg);
 8812 %}
 8813 
 8814 
 8815 // Convert oop pointer into compressed form
 8816 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8817   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8818   match(Set dst (EncodeP src));
 8819   effect(KILL cr);
 8820   ins_cost(INSN_COST * 3);
 8821   format %{ "encode_heap_oop $dst, $src" %}
 8822   ins_encode %{
 8823     Register s = $src$$Register;
 8824     Register d = $dst$$Register;
 8825     __ encode_heap_oop(d, s);
 8826   %}
 8827   ins_pipe(ialu_reg);
 8828 %}
 8829 
 8830 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8831   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8832   match(Set dst (EncodeP src));
 8833   ins_cost(INSN_COST * 3);
 8834   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8835   ins_encode %{
 8836     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8837   %}
 8838   ins_pipe(ialu_reg);
 8839 %}
 8840 
 8841 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8842   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8843             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8844   match(Set dst (DecodeN src));
 8845   ins_cost(INSN_COST * 3);
 8846   format %{ "decode_heap_oop $dst, $src" %}
 8847   ins_encode %{
 8848     Register s = $src$$Register;
 8849     Register d = $dst$$Register;
 8850     __ decode_heap_oop(d, s);
 8851   %}
 8852   ins_pipe(ialu_reg);
 8853 %}
 8854 
 8855 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8856   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8857             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8858   match(Set dst (DecodeN src));
 8859   ins_cost(INSN_COST * 3);
 8860   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8861   ins_encode %{
 8862     Register s = $src$$Register;
 8863     Register d = $dst$$Register;
 8864     __ decode_heap_oop_not_null(d, s);
 8865   %}
 8866   ins_pipe(ialu_reg);
 8867 %}
 8868 
 8869 // n.b. AArch64 implementations of encode_klass_not_null and
 8870 // decode_klass_not_null do not modify the flags register so, unlike
 8871 // Intel, we don't kill CR as a side effect here
 8872 
 8873 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8874   match(Set dst (EncodePKlass src));
 8875 
 8876   ins_cost(INSN_COST * 3);
 8877   format %{ "encode_klass_not_null $dst,$src" %}
 8878 
 8879   ins_encode %{
 8880     Register src_reg = as_Register($src$$reg);
 8881     Register dst_reg = as_Register($dst$$reg);
 8882     __ encode_klass_not_null(dst_reg, src_reg);
 8883   %}
 8884 
 8885    ins_pipe(ialu_reg);
 8886 %}
 8887 
 8888 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8889   match(Set dst (DecodeNKlass src));
 8890 
 8891   ins_cost(INSN_COST * 3);
 8892   format %{ "decode_klass_not_null $dst,$src" %}
 8893 
 8894   ins_encode %{
 8895     Register src_reg = as_Register($src$$reg);
 8896     Register dst_reg = as_Register($dst$$reg);
 8897     if (dst_reg != src_reg) {
 8898       __ decode_klass_not_null(dst_reg, src_reg);
 8899     } else {
 8900       __ decode_klass_not_null(dst_reg);
 8901     }
 8902   %}
 8903 
 8904    ins_pipe(ialu_reg);
 8905 %}
 8906 
 8907 instruct checkCastPP(iRegPNoSp dst)
 8908 %{
 8909   match(Set dst (CheckCastPP dst));
 8910 
 8911   size(0);
 8912   format %{ "# checkcastPP of $dst" %}
 8913   ins_encode(/* empty encoding */);
 8914   ins_pipe(pipe_class_empty);
 8915 %}
 8916 
 8917 instruct castPP(iRegPNoSp dst)
 8918 %{
 8919   match(Set dst (CastPP dst));
 8920 
 8921   size(0);
 8922   format %{ "# castPP of $dst" %}
 8923   ins_encode(/* empty encoding */);
 8924   ins_pipe(pipe_class_empty);
 8925 %}
 8926 
 8927 instruct castII(iRegI dst)
 8928 %{
 8929   match(Set dst (CastII dst));
 8930 
 8931   size(0);
 8932   format %{ "# castII of $dst" %}
 8933   ins_encode(/* empty encoding */);
 8934   ins_cost(0);
 8935   ins_pipe(pipe_class_empty);
 8936 %}
 8937 
 8938 instruct castLL(iRegL dst)
 8939 %{
 8940   match(Set dst (CastLL dst));
 8941 
 8942   size(0);
 8943   format %{ "# castLL of $dst" %}
 8944   ins_encode(/* empty encoding */);
 8945   ins_cost(0);
 8946   ins_pipe(pipe_class_empty);
 8947 %}
 8948 
 8949 instruct castFF(vRegF dst)
 8950 %{
 8951   match(Set dst (CastFF dst));
 8952 
 8953   size(0);
 8954   format %{ "# castFF of $dst" %}
 8955   ins_encode(/* empty encoding */);
 8956   ins_cost(0);
 8957   ins_pipe(pipe_class_empty);
 8958 %}
 8959 
 8960 instruct castDD(vRegD dst)
 8961 %{
 8962   match(Set dst (CastDD dst));
 8963 
 8964   size(0);
 8965   format %{ "# castDD of $dst" %}
 8966   ins_encode(/* empty encoding */);
 8967   ins_cost(0);
 8968   ins_pipe(pipe_class_empty);
 8969 %}
 8970 
 8971 instruct castVVD(vecD dst)
 8972 %{
 8973   match(Set dst (CastVV dst));
 8974 
 8975   size(0);
 8976   format %{ "# castVV of $dst" %}
 8977   ins_encode(/* empty encoding */);
 8978   ins_cost(0);
 8979   ins_pipe(pipe_class_empty);
 8980 %}
 8981 
 8982 instruct castVVX(vecX dst)
 8983 %{
 8984   match(Set dst (CastVV dst));
 8985 
 8986   size(0);
 8987   format %{ "# castVV of $dst" %}
 8988   ins_encode(/* empty encoding */);
 8989   ins_cost(0);
 8990   ins_pipe(pipe_class_empty);
 8991 %}
 8992 
 8993 instruct castVV(vReg dst)
 8994 %{
 8995   match(Set dst (CastVV dst));
 8996 
 8997   size(0);
 8998   format %{ "# castVV of $dst" %}
 8999   ins_encode(/* empty encoding */);
 9000   ins_cost(0);
 9001   ins_pipe(pipe_class_empty);
 9002 %}
 9003 
 9004 // ============================================================================
 9005 // Atomic operation instructions
 9006 //
 9007 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9008 // Store{PIL}Conditional instructions using a normal load for the
 9009 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9010 //
 9011 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9012 // pair to lock object allocations from Eden space when not using
 9013 // TLABs.
 9014 //
 9015 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9016 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9017 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9018 // only for 64-bit.
 9019 //
 9020 // We implement LoadPLocked and StorePLocked instructions using,
 9021 // respectively the AArch64 hw load-exclusive and store-conditional
 9022 // instructions. Whereas we must implement each of
 9023 // Store{IL}Conditional using a CAS which employs a pair of
 9024 // instructions comprising a load-exclusive followed by a
 9025 // store-conditional.
 9026 
 9027 
 9028 // Locked-load (linked load) of the current heap-top
 9029 // used when updating the eden heap top
 9030 // implemented using ldaxr on AArch64
 9031 
 9032 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9033 %{
 9034   match(Set dst (LoadPLocked mem));
 9035 
 9036   ins_cost(VOLATILE_REF_COST);
 9037 
 9038   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9039 
 9040   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9041 
 9042   ins_pipe(pipe_serial);
 9043 %}
 9044 
 9045 // Conditional-store of the updated heap-top.
 9046 // Used during allocation of the shared heap.
 9047 // Sets flag (EQ) on success.
 9048 // implemented using stlxr on AArch64.
 9049 
 9050 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9051 %{
 9052   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9053 
 9054   ins_cost(VOLATILE_REF_COST);
 9055 
 9056  // TODO
 9057  // do we need to do a store-conditional release or can we just use a
 9058  // plain store-conditional?
 9059 
 9060   format %{
 9061     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9062     "cmpw rscratch1, zr\t# EQ on successful write"
 9063   %}
 9064 
 9065   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9066 
 9067   ins_pipe(pipe_serial);
 9068 %}
 9069 
 9070 
 9071 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
 9072 // when attempting to rebias a lock towards the current thread.  We
 9073 // must use the acquire form of cmpxchg in order to guarantee acquire
 9074 // semantics in this case.
 9075 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9076 %{
 9077   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9078 
 9079   ins_cost(VOLATILE_REF_COST);
 9080 
 9081   format %{
 9082     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9083     "cmpw rscratch1, zr\t# EQ on successful write"
 9084   %}
 9085 
 9086   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9087 
 9088   ins_pipe(pipe_slow);
 9089 %}
 9090 
 9091 // storeIConditional also has acquire semantics, for no better reason
 9092 // than matching storeLConditional.  At the time of writing this
 9093 // comment storeIConditional was not used anywhere by AArch64.
 9094 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9095 %{
 9096   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9097 
 9098   ins_cost(VOLATILE_REF_COST);
 9099 
 9100   format %{
 9101     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9102     "cmpw rscratch1, zr\t# EQ on successful write"
 9103   %}
 9104 
 9105   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9106 
 9107   ins_pipe(pipe_slow);
 9108 %}
 9109 
 9110 // standard CompareAndSwapX when we are using barriers
 9111 // these have higher priority than the rules selected by a predicate
 9112 
 9113 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9114 // can't match them
 9115 
 9116 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9117 
 9118   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9119   ins_cost(2 * VOLATILE_REF_COST);
 9120 
 9121   effect(KILL cr);
 9122 
 9123   format %{
 9124     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9125     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9126   %}
 9127 
 9128   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9129             aarch64_enc_cset_eq(res));
 9130 
 9131   ins_pipe(pipe_slow);
 9132 %}
 9133 
 9134 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9135 
 9136   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9137   ins_cost(2 * VOLATILE_REF_COST);
 9138 
 9139   effect(KILL cr);
 9140 
 9141   format %{
 9142     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9143     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9144   %}
 9145 
 9146   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9147             aarch64_enc_cset_eq(res));
 9148 
 9149   ins_pipe(pipe_slow);
 9150 %}
 9151 
 9152 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9153 
 9154   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9155   ins_cost(2 * VOLATILE_REF_COST);
 9156 
 9157   effect(KILL cr);
 9158 
 9159  format %{
 9160     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9161     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9162  %}
 9163 
 9164  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9165             aarch64_enc_cset_eq(res));
 9166 
 9167   ins_pipe(pipe_slow);
 9168 %}
 9169 
 9170 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9171 
 9172   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9173   ins_cost(2 * VOLATILE_REF_COST);
 9174 
 9175   effect(KILL cr);
 9176 
 9177  format %{
 9178     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9179     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9180  %}
 9181 
 9182  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9183             aarch64_enc_cset_eq(res));
 9184 
 9185   ins_pipe(pipe_slow);
 9186 %}
 9187 
 9188 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9189 
 9190   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9191   predicate(n->as_LoadStore()->barrier_data() == 0);
 9192   ins_cost(2 * VOLATILE_REF_COST);
 9193 
 9194   effect(KILL cr);
 9195 
 9196  format %{
 9197     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9198     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9199  %}
 9200 
 9201  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9202             aarch64_enc_cset_eq(res));
 9203 
 9204   ins_pipe(pipe_slow);
 9205 %}
 9206 
 9207 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9208 
 9209   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9210   ins_cost(2 * VOLATILE_REF_COST);
 9211 
 9212   effect(KILL cr);
 9213 
 9214  format %{
 9215     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9216     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9217  %}
 9218 
 9219  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9220             aarch64_enc_cset_eq(res));
 9221 
 9222   ins_pipe(pipe_slow);
 9223 %}
 9224 
 9225 // alternative CompareAndSwapX when we are eliding barriers
 9226 
 9227 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9228 
 9229   predicate(needs_acquiring_load_exclusive(n));
 9230   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9231   ins_cost(VOLATILE_REF_COST);
 9232 
 9233   effect(KILL cr);
 9234 
 9235   format %{
 9236     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9237     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9238   %}
 9239 
 9240   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9241             aarch64_enc_cset_eq(res));
 9242 
 9243   ins_pipe(pipe_slow);
 9244 %}
 9245 
 9246 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9247 
 9248   predicate(needs_acquiring_load_exclusive(n));
 9249   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9250   ins_cost(VOLATILE_REF_COST);
 9251 
 9252   effect(KILL cr);
 9253 
 9254   format %{
 9255     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9256     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9257   %}
 9258 
 9259   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9260             aarch64_enc_cset_eq(res));
 9261 
 9262   ins_pipe(pipe_slow);
 9263 %}
 9264 
 9265 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9266 
 9267   predicate(needs_acquiring_load_exclusive(n));
 9268   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9269   ins_cost(VOLATILE_REF_COST);
 9270 
 9271   effect(KILL cr);
 9272 
 9273  format %{
 9274     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9275     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9276  %}
 9277 
 9278  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9279             aarch64_enc_cset_eq(res));
 9280 
 9281   ins_pipe(pipe_slow);
 9282 %}
 9283 
 9284 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9285 
 9286   predicate(needs_acquiring_load_exclusive(n));
 9287   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9288   ins_cost(VOLATILE_REF_COST);
 9289 
 9290   effect(KILL cr);
 9291 
 9292  format %{
 9293     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9294     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9295  %}
 9296 
 9297  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9298             aarch64_enc_cset_eq(res));
 9299 
 9300   ins_pipe(pipe_slow);
 9301 %}
 9302 
 9303 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9304 
 9305   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9306   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9307   ins_cost(VOLATILE_REF_COST);
 9308 
 9309   effect(KILL cr);
 9310 
 9311  format %{
 9312     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9313     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9314  %}
 9315 
 9316  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9317             aarch64_enc_cset_eq(res));
 9318 
 9319   ins_pipe(pipe_slow);
 9320 %}
 9321 
 9322 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9323 
 9324   predicate(needs_acquiring_load_exclusive(n));
 9325   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9326   ins_cost(VOLATILE_REF_COST);
 9327 
 9328   effect(KILL cr);
 9329 
 9330  format %{
 9331     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9332     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9333  %}
 9334 
 9335  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9336             aarch64_enc_cset_eq(res));
 9337 
 9338   ins_pipe(pipe_slow);
 9339 %}
 9340 
 9341 
 9342 // ---------------------------------------------------------------------
 9343 
 9344 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9345 
 9346 // Sundry CAS operations.  Note that release is always true,
 9347 // regardless of the memory ordering of the CAS.  This is because we
 9348 // need the volatile case to be sequentially consistent but there is
 9349 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9350 // can't check the type of memory ordering here, so we always emit a
 9351 // STLXR.
 9352 
 9353 // This section is generated from aarch64_ad_cas.m4
 9354 
 9355 
 9356 
 9357 // This pattern is generated automatically from cas.m4.
 9358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9359 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9360 
 9361   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9362   ins_cost(2 * VOLATILE_REF_COST);
 9363   effect(TEMP_DEF res, KILL cr);
 9364   format %{
 9365     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9366   %}
 9367   ins_encode %{
 9368     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9369                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9370                /*weak*/ false, $res$$Register);
 9371     __ sxtbw($res$$Register, $res$$Register);
 9372   %}
 9373   ins_pipe(pipe_slow);
 9374 %}
 9375 
 9376 // This pattern is generated automatically from cas.m4.
 9377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9378 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9379 
 9380   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9381   ins_cost(2 * VOLATILE_REF_COST);
 9382   effect(TEMP_DEF res, KILL cr);
 9383   format %{
 9384     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9385   %}
 9386   ins_encode %{
 9387     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9388                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9389                /*weak*/ false, $res$$Register);
 9390     __ sxthw($res$$Register, $res$$Register);
 9391   %}
 9392   ins_pipe(pipe_slow);
 9393 %}
 9394 
 9395 // This pattern is generated automatically from cas.m4.
 9396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9397 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9398 
 9399   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9400   ins_cost(2 * VOLATILE_REF_COST);
 9401   effect(TEMP_DEF res, KILL cr);
 9402   format %{
 9403     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9404   %}
 9405   ins_encode %{
 9406     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9407                Assembler::word, /*acquire*/ false, /*release*/ true,
 9408                /*weak*/ false, $res$$Register);
 9409   %}
 9410   ins_pipe(pipe_slow);
 9411 %}
 9412 
 9413 // This pattern is generated automatically from cas.m4.
 9414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9415 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9416 
 9417   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9418   ins_cost(2 * VOLATILE_REF_COST);
 9419   effect(TEMP_DEF res, KILL cr);
 9420   format %{
 9421     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9422   %}
 9423   ins_encode %{
 9424     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9425                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9426                /*weak*/ false, $res$$Register);
 9427   %}
 9428   ins_pipe(pipe_slow);
 9429 %}
 9430 
 9431 // This pattern is generated automatically from cas.m4.
 9432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9433 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9434 
 9435   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9436   ins_cost(2 * VOLATILE_REF_COST);
 9437   effect(TEMP_DEF res, KILL cr);
 9438   format %{
 9439     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9440   %}
 9441   ins_encode %{
 9442     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9443                Assembler::word, /*acquire*/ false, /*release*/ true,
 9444                /*weak*/ false, $res$$Register);
 9445   %}
 9446   ins_pipe(pipe_slow);
 9447 %}
 9448 
 9449 // This pattern is generated automatically from cas.m4.
 9450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9451 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9452   predicate(n->as_LoadStore()->barrier_data() == 0);
 9453   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9454   ins_cost(2 * VOLATILE_REF_COST);
 9455   effect(TEMP_DEF res, KILL cr);
 9456   format %{
 9457     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9458   %}
 9459   ins_encode %{
 9460     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9461                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9462                /*weak*/ false, $res$$Register);
 9463   %}
 9464   ins_pipe(pipe_slow);
 9465 %}
 9466 
 9467 // This pattern is generated automatically from cas.m4.
 9468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9469 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9470   predicate(needs_acquiring_load_exclusive(n));
 9471   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9472   ins_cost(VOLATILE_REF_COST);
 9473   effect(TEMP_DEF res, KILL cr);
 9474   format %{
 9475     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9476   %}
 9477   ins_encode %{
 9478     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9479                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9480                /*weak*/ false, $res$$Register);
 9481     __ sxtbw($res$$Register, $res$$Register);
 9482   %}
 9483   ins_pipe(pipe_slow);
 9484 %}
 9485 
 9486 // This pattern is generated automatically from cas.m4.
 9487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9488 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9489   predicate(needs_acquiring_load_exclusive(n));
 9490   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9491   ins_cost(VOLATILE_REF_COST);
 9492   effect(TEMP_DEF res, KILL cr);
 9493   format %{
 9494     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9495   %}
 9496   ins_encode %{
 9497     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9498                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9499                /*weak*/ false, $res$$Register);
 9500     __ sxthw($res$$Register, $res$$Register);
 9501   %}
 9502   ins_pipe(pipe_slow);
 9503 %}
 9504 
 9505 // This pattern is generated automatically from cas.m4.
 9506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9507 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9508   predicate(needs_acquiring_load_exclusive(n));
 9509   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9510   ins_cost(VOLATILE_REF_COST);
 9511   effect(TEMP_DEF res, KILL cr);
 9512   format %{
 9513     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9514   %}
 9515   ins_encode %{
 9516     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9517                Assembler::word, /*acquire*/ true, /*release*/ true,
 9518                /*weak*/ false, $res$$Register);
 9519   %}
 9520   ins_pipe(pipe_slow);
 9521 %}
 9522 
 9523 // This pattern is generated automatically from cas.m4.
 9524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9525 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9526   predicate(needs_acquiring_load_exclusive(n));
 9527   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9528   ins_cost(VOLATILE_REF_COST);
 9529   effect(TEMP_DEF res, KILL cr);
 9530   format %{
 9531     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9532   %}
 9533   ins_encode %{
 9534     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9535                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9536                /*weak*/ false, $res$$Register);
 9537   %}
 9538   ins_pipe(pipe_slow);
 9539 %}
 9540 
 9541 // This pattern is generated automatically from cas.m4.
 9542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9543 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9544   predicate(needs_acquiring_load_exclusive(n));
 9545   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9546   ins_cost(VOLATILE_REF_COST);
 9547   effect(TEMP_DEF res, KILL cr);
 9548   format %{
 9549     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9550   %}
 9551   ins_encode %{
 9552     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9553                Assembler::word, /*acquire*/ true, /*release*/ true,
 9554                /*weak*/ false, $res$$Register);
 9555   %}
 9556   ins_pipe(pipe_slow);
 9557 %}
 9558 
 9559 // This pattern is generated automatically from cas.m4.
 9560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9561 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9562   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9563   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9564   ins_cost(VOLATILE_REF_COST);
 9565   effect(TEMP_DEF res, KILL cr);
 9566   format %{
 9567     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9568   %}
 9569   ins_encode %{
 9570     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9571                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9572                /*weak*/ false, $res$$Register);
 9573   %}
 9574   ins_pipe(pipe_slow);
 9575 %}
 9576 
 9577 // This pattern is generated automatically from cas.m4.
 9578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9579 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9580 
 9581   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9582   ins_cost(2 * VOLATILE_REF_COST);
 9583   effect(KILL cr);
 9584   format %{
 9585     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9586     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9587   %}
 9588   ins_encode %{
 9589     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9590                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9591                /*weak*/ true, noreg);
 9592     __ csetw($res$$Register, Assembler::EQ);
 9593   %}
 9594   ins_pipe(pipe_slow);
 9595 %}
 9596 
 9597 // This pattern is generated automatically from cas.m4.
 9598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9599 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9600 
 9601   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9602   ins_cost(2 * VOLATILE_REF_COST);
 9603   effect(KILL cr);
 9604   format %{
 9605     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9606     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9607   %}
 9608   ins_encode %{
 9609     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9610                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9611                /*weak*/ true, noreg);
 9612     __ csetw($res$$Register, Assembler::EQ);
 9613   %}
 9614   ins_pipe(pipe_slow);
 9615 %}
 9616 
 9617 // This pattern is generated automatically from cas.m4.
 9618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9619 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9620 
 9621   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9622   ins_cost(2 * VOLATILE_REF_COST);
 9623   effect(KILL cr);
 9624   format %{
 9625     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9626     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9627   %}
 9628   ins_encode %{
 9629     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9630                Assembler::word, /*acquire*/ false, /*release*/ true,
 9631                /*weak*/ true, noreg);
 9632     __ csetw($res$$Register, Assembler::EQ);
 9633   %}
 9634   ins_pipe(pipe_slow);
 9635 %}
 9636 
 9637 // This pattern is generated automatically from cas.m4.
 9638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9639 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9640 
 9641   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9642   ins_cost(2 * VOLATILE_REF_COST);
 9643   effect(KILL cr);
 9644   format %{
 9645     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9646     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9647   %}
 9648   ins_encode %{
 9649     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9650                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9651                /*weak*/ true, noreg);
 9652     __ csetw($res$$Register, Assembler::EQ);
 9653   %}
 9654   ins_pipe(pipe_slow);
 9655 %}
 9656 
 9657 // This pattern is generated automatically from cas.m4.
 9658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9659 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9660 
 9661   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9662   ins_cost(2 * VOLATILE_REF_COST);
 9663   effect(KILL cr);
 9664   format %{
 9665     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9666     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9667   %}
 9668   ins_encode %{
 9669     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9670                Assembler::word, /*acquire*/ false, /*release*/ true,
 9671                /*weak*/ true, noreg);
 9672     __ csetw($res$$Register, Assembler::EQ);
 9673   %}
 9674   ins_pipe(pipe_slow);
 9675 %}
 9676 
 9677 // This pattern is generated automatically from cas.m4.
 9678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9679 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9680   predicate(n->as_LoadStore()->barrier_data() == 0);
 9681   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9682   ins_cost(2 * VOLATILE_REF_COST);
 9683   effect(KILL cr);
 9684   format %{
 9685     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9686     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9687   %}
 9688   ins_encode %{
 9689     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9690                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9691                /*weak*/ true, noreg);
 9692     __ csetw($res$$Register, Assembler::EQ);
 9693   %}
 9694   ins_pipe(pipe_slow);
 9695 %}
 9696 
 9697 // This pattern is generated automatically from cas.m4.
 9698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9699 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9700   predicate(needs_acquiring_load_exclusive(n));
 9701   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9702   ins_cost(VOLATILE_REF_COST);
 9703   effect(KILL cr);
 9704   format %{
 9705     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9706     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9707   %}
 9708   ins_encode %{
 9709     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9710                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9711                /*weak*/ true, noreg);
 9712     __ csetw($res$$Register, Assembler::EQ);
 9713   %}
 9714   ins_pipe(pipe_slow);
 9715 %}
 9716 
 9717 // This pattern is generated automatically from cas.m4.
 9718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9719 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9720   predicate(needs_acquiring_load_exclusive(n));
 9721   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9722   ins_cost(VOLATILE_REF_COST);
 9723   effect(KILL cr);
 9724   format %{
 9725     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9726     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9727   %}
 9728   ins_encode %{
 9729     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9730                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9731                /*weak*/ true, noreg);
 9732     __ csetw($res$$Register, Assembler::EQ);
 9733   %}
 9734   ins_pipe(pipe_slow);
 9735 %}
 9736 
 9737 // This pattern is generated automatically from cas.m4.
 9738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9739 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9740   predicate(needs_acquiring_load_exclusive(n));
 9741   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9742   ins_cost(VOLATILE_REF_COST);
 9743   effect(KILL cr);
 9744   format %{
 9745     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9746     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9747   %}
 9748   ins_encode %{
 9749     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9750                Assembler::word, /*acquire*/ true, /*release*/ true,
 9751                /*weak*/ true, noreg);
 9752     __ csetw($res$$Register, Assembler::EQ);
 9753   %}
 9754   ins_pipe(pipe_slow);
 9755 %}
 9756 
 9757 // This pattern is generated automatically from cas.m4.
 9758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9759 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9760   predicate(needs_acquiring_load_exclusive(n));
 9761   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9762   ins_cost(VOLATILE_REF_COST);
 9763   effect(KILL cr);
 9764   format %{
 9765     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9766     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9767   %}
 9768   ins_encode %{
 9769     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9770                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9771                /*weak*/ true, noreg);
 9772     __ csetw($res$$Register, Assembler::EQ);
 9773   %}
 9774   ins_pipe(pipe_slow);
 9775 %}
 9776 
 9777 // This pattern is generated automatically from cas.m4.
 9778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9779 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9780   predicate(needs_acquiring_load_exclusive(n));
 9781   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9782   ins_cost(VOLATILE_REF_COST);
 9783   effect(KILL cr);
 9784   format %{
 9785     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9786     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9787   %}
 9788   ins_encode %{
 9789     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9790                Assembler::word, /*acquire*/ true, /*release*/ true,
 9791                /*weak*/ true, noreg);
 9792     __ csetw($res$$Register, Assembler::EQ);
 9793   %}
 9794   ins_pipe(pipe_slow);
 9795 %}
 9796 
 9797 // This pattern is generated automatically from cas.m4.
 9798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9799 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9800   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9801   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9802   ins_cost(VOLATILE_REF_COST);
 9803   effect(KILL cr);
 9804   format %{
 9805     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9806     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9807   %}
 9808   ins_encode %{
 9809     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9810                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9811                /*weak*/ true, noreg);
 9812     __ csetw($res$$Register, Assembler::EQ);
 9813   %}
 9814   ins_pipe(pipe_slow);
 9815 %}
 9816 
 9817 // END This section of the file is automatically generated. Do not edit --------------
 9818 // ---------------------------------------------------------------------
 9819 
 9820 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9821   match(Set prev (GetAndSetI mem newv));
 9822   ins_cost(2 * VOLATILE_REF_COST);
 9823   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9824   ins_encode %{
 9825     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9826   %}
 9827   ins_pipe(pipe_serial);
 9828 %}
 9829 
 9830 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9831   match(Set prev (GetAndSetL mem newv));
 9832   ins_cost(2 * VOLATILE_REF_COST);
 9833   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9834   ins_encode %{
 9835     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9836   %}
 9837   ins_pipe(pipe_serial);
 9838 %}
 9839 
 9840 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9841   match(Set prev (GetAndSetN mem newv));
 9842   ins_cost(2 * VOLATILE_REF_COST);
 9843   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9844   ins_encode %{
 9845     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9846   %}
 9847   ins_pipe(pipe_serial);
 9848 %}
 9849 
 9850 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9851   predicate(n->as_LoadStore()->barrier_data() == 0);
 9852   match(Set prev (GetAndSetP mem newv));
 9853   ins_cost(2 * VOLATILE_REF_COST);
 9854   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9855   ins_encode %{
 9856     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9857   %}
 9858   ins_pipe(pipe_serial);
 9859 %}
 9860 
 9861 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9862   predicate(needs_acquiring_load_exclusive(n));
 9863   match(Set prev (GetAndSetI mem newv));
 9864   ins_cost(VOLATILE_REF_COST);
 9865   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9866   ins_encode %{
 9867     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9868   %}
 9869   ins_pipe(pipe_serial);
 9870 %}
 9871 
 9872 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9873   predicate(needs_acquiring_load_exclusive(n));
 9874   match(Set prev (GetAndSetL mem newv));
 9875   ins_cost(VOLATILE_REF_COST);
 9876   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9877   ins_encode %{
 9878     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9879   %}
 9880   ins_pipe(pipe_serial);
 9881 %}
 9882 
 9883 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9884   predicate(needs_acquiring_load_exclusive(n));
 9885   match(Set prev (GetAndSetN mem newv));
 9886   ins_cost(VOLATILE_REF_COST);
 9887   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9888   ins_encode %{
 9889     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9890   %}
 9891   ins_pipe(pipe_serial);
 9892 %}
 9893 
 9894 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9895   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9896   match(Set prev (GetAndSetP mem newv));
 9897   ins_cost(VOLATILE_REF_COST);
 9898   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9899   ins_encode %{
 9900     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9901   %}
 9902   ins_pipe(pipe_serial);
 9903 %}
 9904 
 9905 
 9906 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9907   match(Set newval (GetAndAddL mem incr));
 9908   ins_cost(2 * VOLATILE_REF_COST + 1);
 9909   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9910   ins_encode %{
 9911     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9912   %}
 9913   ins_pipe(pipe_serial);
 9914 %}
 9915 
 9916 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9917   predicate(n->as_LoadStore()->result_not_used());
 9918   match(Set dummy (GetAndAddL mem incr));
 9919   ins_cost(2 * VOLATILE_REF_COST);
 9920   format %{ "get_and_addL [$mem], $incr" %}
 9921   ins_encode %{
 9922     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9923   %}
 9924   ins_pipe(pipe_serial);
 9925 %}
 9926 
 9927 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9928   match(Set newval (GetAndAddL mem incr));
 9929   ins_cost(2 * VOLATILE_REF_COST + 1);
 9930   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9931   ins_encode %{
 9932     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9933   %}
 9934   ins_pipe(pipe_serial);
 9935 %}
 9936 
 9937 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9938   predicate(n->as_LoadStore()->result_not_used());
 9939   match(Set dummy (GetAndAddL mem incr));
 9940   ins_cost(2 * VOLATILE_REF_COST);
 9941   format %{ "get_and_addL [$mem], $incr" %}
 9942   ins_encode %{
 9943     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9944   %}
 9945   ins_pipe(pipe_serial);
 9946 %}
 9947 
 9948 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9949   match(Set newval (GetAndAddI mem incr));
 9950   ins_cost(2 * VOLATILE_REF_COST + 1);
 9951   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9952   ins_encode %{
 9953     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9954   %}
 9955   ins_pipe(pipe_serial);
 9956 %}
 9957 
 9958 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9959   predicate(n->as_LoadStore()->result_not_used());
 9960   match(Set dummy (GetAndAddI mem incr));
 9961   ins_cost(2 * VOLATILE_REF_COST);
 9962   format %{ "get_and_addI [$mem], $incr" %}
 9963   ins_encode %{
 9964     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9965   %}
 9966   ins_pipe(pipe_serial);
 9967 %}
 9968 
 9969 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9970   match(Set newval (GetAndAddI mem incr));
 9971   ins_cost(2 * VOLATILE_REF_COST + 1);
 9972   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9973   ins_encode %{
 9974     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9975   %}
 9976   ins_pipe(pipe_serial);
 9977 %}
 9978 
 9979 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9980   predicate(n->as_LoadStore()->result_not_used());
 9981   match(Set dummy (GetAndAddI mem incr));
 9982   ins_cost(2 * VOLATILE_REF_COST);
 9983   format %{ "get_and_addI [$mem], $incr" %}
 9984   ins_encode %{
 9985     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9986   %}
 9987   ins_pipe(pipe_serial);
 9988 %}
 9989 
 9990 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9991   predicate(needs_acquiring_load_exclusive(n));
 9992   match(Set newval (GetAndAddL mem incr));
 9993   ins_cost(VOLATILE_REF_COST + 1);
 9994   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9995   ins_encode %{
 9996     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9997   %}
 9998   ins_pipe(pipe_serial);
 9999 %}
10000 
10001 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10002   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10003   match(Set dummy (GetAndAddL mem incr));
10004   ins_cost(VOLATILE_REF_COST);
10005   format %{ "get_and_addL_acq [$mem], $incr" %}
10006   ins_encode %{
10007     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10008   %}
10009   ins_pipe(pipe_serial);
10010 %}
10011 
10012 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10013   predicate(needs_acquiring_load_exclusive(n));
10014   match(Set newval (GetAndAddL mem incr));
10015   ins_cost(VOLATILE_REF_COST + 1);
10016   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10017   ins_encode %{
10018     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10019   %}
10020   ins_pipe(pipe_serial);
10021 %}
10022 
10023 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10024   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10025   match(Set dummy (GetAndAddL mem incr));
10026   ins_cost(VOLATILE_REF_COST);
10027   format %{ "get_and_addL_acq [$mem], $incr" %}
10028   ins_encode %{
10029     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10030   %}
10031   ins_pipe(pipe_serial);
10032 %}
10033 
10034 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10035   predicate(needs_acquiring_load_exclusive(n));
10036   match(Set newval (GetAndAddI mem incr));
10037   ins_cost(VOLATILE_REF_COST + 1);
10038   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10039   ins_encode %{
10040     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10041   %}
10042   ins_pipe(pipe_serial);
10043 %}
10044 
10045 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10046   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10047   match(Set dummy (GetAndAddI mem incr));
10048   ins_cost(VOLATILE_REF_COST);
10049   format %{ "get_and_addI_acq [$mem], $incr" %}
10050   ins_encode %{
10051     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10052   %}
10053   ins_pipe(pipe_serial);
10054 %}
10055 
10056 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10057   predicate(needs_acquiring_load_exclusive(n));
10058   match(Set newval (GetAndAddI mem incr));
10059   ins_cost(VOLATILE_REF_COST + 1);
10060   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10061   ins_encode %{
10062     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10063   %}
10064   ins_pipe(pipe_serial);
10065 %}
10066 
10067 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10068   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10069   match(Set dummy (GetAndAddI mem incr));
10070   ins_cost(VOLATILE_REF_COST);
10071   format %{ "get_and_addI_acq [$mem], $incr" %}
10072   ins_encode %{
10073     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10074   %}
10075   ins_pipe(pipe_serial);
10076 %}
10077 
10078 // Manifest a CmpL result in an integer register.
10079 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10080 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10081 %{
10082   match(Set dst (CmpL3 src1 src2));
10083   effect(KILL flags);
10084 
10085   ins_cost(INSN_COST * 6);
10086   format %{
10087       "cmp $src1, $src2"
10088       "csetw $dst, ne"
10089       "cnegw $dst, lt"
10090   %}
10091   // format %{ "CmpL3 $dst, $src1, $src2" %}
10092   ins_encode %{
10093     __ cmp($src1$$Register, $src2$$Register);
10094     __ csetw($dst$$Register, Assembler::NE);
10095     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10096   %}
10097 
10098   ins_pipe(pipe_class_default);
10099 %}
10100 
10101 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10102 %{
10103   match(Set dst (CmpL3 src1 src2));
10104   effect(KILL flags);
10105 
10106   ins_cost(INSN_COST * 6);
10107   format %{
10108       "cmp $src1, $src2"
10109       "csetw $dst, ne"
10110       "cnegw $dst, lt"
10111   %}
10112   ins_encode %{
10113     int32_t con = (int32_t)$src2$$constant;
10114      if (con < 0) {
10115       __ adds(zr, $src1$$Register, -con);
10116     } else {
10117       __ subs(zr, $src1$$Register, con);
10118     }
10119     __ csetw($dst$$Register, Assembler::NE);
10120     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10121   %}
10122 
10123   ins_pipe(pipe_class_default);
10124 %}
10125 
10126 // ============================================================================
10127 // Conditional Move Instructions
10128 
10129 // n.b. we have identical rules for both a signed compare op (cmpOp)
10130 // and an unsigned compare op (cmpOpU). it would be nice if we could
10131 // define an op class which merged both inputs and use it to type the
10132 // argument to a single rule. unfortunatelyt his fails because the
10133 // opclass does not live up to the COND_INTER interface of its
10134 // component operands. When the generic code tries to negate the
10135 // operand it ends up running the generci Machoper::negate method
10136 // which throws a ShouldNotHappen. So, we have to provide two flavours
10137 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10138 
10139 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10140   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10141 
10142   ins_cost(INSN_COST * 2);
10143   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10144 
10145   ins_encode %{
10146     __ cselw(as_Register($dst$$reg),
10147              as_Register($src2$$reg),
10148              as_Register($src1$$reg),
10149              (Assembler::Condition)$cmp$$cmpcode);
10150   %}
10151 
10152   ins_pipe(icond_reg_reg);
10153 %}
10154 
10155 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10156   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10157 
10158   ins_cost(INSN_COST * 2);
10159   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10160 
10161   ins_encode %{
10162     __ cselw(as_Register($dst$$reg),
10163              as_Register($src2$$reg),
10164              as_Register($src1$$reg),
10165              (Assembler::Condition)$cmp$$cmpcode);
10166   %}
10167 
10168   ins_pipe(icond_reg_reg);
10169 %}
10170 
10171 // special cases where one arg is zero
10172 
10173 // n.b. this is selected in preference to the rule above because it
10174 // avoids loading constant 0 into a source register
10175 
10176 // TODO
10177 // we ought only to be able to cull one of these variants as the ideal
10178 // transforms ought always to order the zero consistently (to left/right?)
10179 
10180 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10181   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10182 
10183   ins_cost(INSN_COST * 2);
10184   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10185 
10186   ins_encode %{
10187     __ cselw(as_Register($dst$$reg),
10188              as_Register($src$$reg),
10189              zr,
10190              (Assembler::Condition)$cmp$$cmpcode);
10191   %}
10192 
10193   ins_pipe(icond_reg);
10194 %}
10195 
10196 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10197   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10198 
10199   ins_cost(INSN_COST * 2);
10200   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10201 
10202   ins_encode %{
10203     __ cselw(as_Register($dst$$reg),
10204              as_Register($src$$reg),
10205              zr,
10206              (Assembler::Condition)$cmp$$cmpcode);
10207   %}
10208 
10209   ins_pipe(icond_reg);
10210 %}
10211 
10212 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10213   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10214 
10215   ins_cost(INSN_COST * 2);
10216   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10217 
10218   ins_encode %{
10219     __ cselw(as_Register($dst$$reg),
10220              zr,
10221              as_Register($src$$reg),
10222              (Assembler::Condition)$cmp$$cmpcode);
10223   %}
10224 
10225   ins_pipe(icond_reg);
10226 %}
10227 
10228 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10229   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10230 
10231   ins_cost(INSN_COST * 2);
10232   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10233 
10234   ins_encode %{
10235     __ cselw(as_Register($dst$$reg),
10236              zr,
10237              as_Register($src$$reg),
10238              (Assembler::Condition)$cmp$$cmpcode);
10239   %}
10240 
10241   ins_pipe(icond_reg);
10242 %}
10243 
10244 // special case for creating a boolean 0 or 1
10245 
10246 // n.b. this is selected in preference to the rule above because it
10247 // avoids loading constants 0 and 1 into a source register
10248 
10249 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10250   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10251 
10252   ins_cost(INSN_COST * 2);
10253   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10254 
10255   ins_encode %{
10256     // equivalently
10257     // cset(as_Register($dst$$reg),
10258     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10259     __ csincw(as_Register($dst$$reg),
10260              zr,
10261              zr,
10262              (Assembler::Condition)$cmp$$cmpcode);
10263   %}
10264 
10265   ins_pipe(icond_none);
10266 %}
10267 
10268 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10269   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10270 
10271   ins_cost(INSN_COST * 2);
10272   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10273 
10274   ins_encode %{
10275     // equivalently
10276     // cset(as_Register($dst$$reg),
10277     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10278     __ csincw(as_Register($dst$$reg),
10279              zr,
10280              zr,
10281              (Assembler::Condition)$cmp$$cmpcode);
10282   %}
10283 
10284   ins_pipe(icond_none);
10285 %}
10286 
10287 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10288   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10289 
10290   ins_cost(INSN_COST * 2);
10291   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10292 
10293   ins_encode %{
10294     __ csel(as_Register($dst$$reg),
10295             as_Register($src2$$reg),
10296             as_Register($src1$$reg),
10297             (Assembler::Condition)$cmp$$cmpcode);
10298   %}
10299 
10300   ins_pipe(icond_reg_reg);
10301 %}
10302 
10303 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10304   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10305 
10306   ins_cost(INSN_COST * 2);
10307   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10308 
10309   ins_encode %{
10310     __ csel(as_Register($dst$$reg),
10311             as_Register($src2$$reg),
10312             as_Register($src1$$reg),
10313             (Assembler::Condition)$cmp$$cmpcode);
10314   %}
10315 
10316   ins_pipe(icond_reg_reg);
10317 %}
10318 
10319 // special cases where one arg is zero
10320 
10321 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10322   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10323 
10324   ins_cost(INSN_COST * 2);
10325   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10326 
10327   ins_encode %{
10328     __ csel(as_Register($dst$$reg),
10329             zr,
10330             as_Register($src$$reg),
10331             (Assembler::Condition)$cmp$$cmpcode);
10332   %}
10333 
10334   ins_pipe(icond_reg);
10335 %}
10336 
10337 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10338   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10339 
10340   ins_cost(INSN_COST * 2);
10341   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10342 
10343   ins_encode %{
10344     __ csel(as_Register($dst$$reg),
10345             zr,
10346             as_Register($src$$reg),
10347             (Assembler::Condition)$cmp$$cmpcode);
10348   %}
10349 
10350   ins_pipe(icond_reg);
10351 %}
10352 
10353 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10354   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10355 
10356   ins_cost(INSN_COST * 2);
10357   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10358 
10359   ins_encode %{
10360     __ csel(as_Register($dst$$reg),
10361             as_Register($src$$reg),
10362             zr,
10363             (Assembler::Condition)$cmp$$cmpcode);
10364   %}
10365 
10366   ins_pipe(icond_reg);
10367 %}
10368 
10369 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10370   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10371 
10372   ins_cost(INSN_COST * 2);
10373   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10374 
10375   ins_encode %{
10376     __ csel(as_Register($dst$$reg),
10377             as_Register($src$$reg),
10378             zr,
10379             (Assembler::Condition)$cmp$$cmpcode);
10380   %}
10381 
10382   ins_pipe(icond_reg);
10383 %}
10384 
10385 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10386   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10387 
10388   ins_cost(INSN_COST * 2);
10389   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10390 
10391   ins_encode %{
10392     __ csel(as_Register($dst$$reg),
10393             as_Register($src2$$reg),
10394             as_Register($src1$$reg),
10395             (Assembler::Condition)$cmp$$cmpcode);
10396   %}
10397 
10398   ins_pipe(icond_reg_reg);
10399 %}
10400 
10401 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10402   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10403 
10404   ins_cost(INSN_COST * 2);
10405   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10406 
10407   ins_encode %{
10408     __ csel(as_Register($dst$$reg),
10409             as_Register($src2$$reg),
10410             as_Register($src1$$reg),
10411             (Assembler::Condition)$cmp$$cmpcode);
10412   %}
10413 
10414   ins_pipe(icond_reg_reg);
10415 %}
10416 
10417 // special cases where one arg is zero
10418 
10419 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10420   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10421 
10422   ins_cost(INSN_COST * 2);
10423   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10424 
10425   ins_encode %{
10426     __ csel(as_Register($dst$$reg),
10427             zr,
10428             as_Register($src$$reg),
10429             (Assembler::Condition)$cmp$$cmpcode);
10430   %}
10431 
10432   ins_pipe(icond_reg);
10433 %}
10434 
10435 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10436   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10437 
10438   ins_cost(INSN_COST * 2);
10439   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10440 
10441   ins_encode %{
10442     __ csel(as_Register($dst$$reg),
10443             zr,
10444             as_Register($src$$reg),
10445             (Assembler::Condition)$cmp$$cmpcode);
10446   %}
10447 
10448   ins_pipe(icond_reg);
10449 %}
10450 
10451 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10452   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10453 
10454   ins_cost(INSN_COST * 2);
10455   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10456 
10457   ins_encode %{
10458     __ csel(as_Register($dst$$reg),
10459             as_Register($src$$reg),
10460             zr,
10461             (Assembler::Condition)$cmp$$cmpcode);
10462   %}
10463 
10464   ins_pipe(icond_reg);
10465 %}
10466 
10467 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10468   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10469 
10470   ins_cost(INSN_COST * 2);
10471   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10472 
10473   ins_encode %{
10474     __ csel(as_Register($dst$$reg),
10475             as_Register($src$$reg),
10476             zr,
10477             (Assembler::Condition)$cmp$$cmpcode);
10478   %}
10479 
10480   ins_pipe(icond_reg);
10481 %}
10482 
10483 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10484   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10485 
10486   ins_cost(INSN_COST * 2);
10487   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10488 
10489   ins_encode %{
10490     __ cselw(as_Register($dst$$reg),
10491              as_Register($src2$$reg),
10492              as_Register($src1$$reg),
10493              (Assembler::Condition)$cmp$$cmpcode);
10494   %}
10495 
10496   ins_pipe(icond_reg_reg);
10497 %}
10498 
10499 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10500   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10501 
10502   ins_cost(INSN_COST * 2);
10503   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10504 
10505   ins_encode %{
10506     __ cselw(as_Register($dst$$reg),
10507              as_Register($src2$$reg),
10508              as_Register($src1$$reg),
10509              (Assembler::Condition)$cmp$$cmpcode);
10510   %}
10511 
10512   ins_pipe(icond_reg_reg);
10513 %}
10514 
10515 // special cases where one arg is zero
10516 
10517 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10518   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10519 
10520   ins_cost(INSN_COST * 2);
10521   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10522 
10523   ins_encode %{
10524     __ cselw(as_Register($dst$$reg),
10525              zr,
10526              as_Register($src$$reg),
10527              (Assembler::Condition)$cmp$$cmpcode);
10528   %}
10529 
10530   ins_pipe(icond_reg);
10531 %}
10532 
10533 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10534   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10535 
10536   ins_cost(INSN_COST * 2);
10537   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10538 
10539   ins_encode %{
10540     __ cselw(as_Register($dst$$reg),
10541              zr,
10542              as_Register($src$$reg),
10543              (Assembler::Condition)$cmp$$cmpcode);
10544   %}
10545 
10546   ins_pipe(icond_reg);
10547 %}
10548 
10549 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10550   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10551 
10552   ins_cost(INSN_COST * 2);
10553   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10554 
10555   ins_encode %{
10556     __ cselw(as_Register($dst$$reg),
10557              as_Register($src$$reg),
10558              zr,
10559              (Assembler::Condition)$cmp$$cmpcode);
10560   %}
10561 
10562   ins_pipe(icond_reg);
10563 %}
10564 
10565 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10566   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10567 
10568   ins_cost(INSN_COST * 2);
10569   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10570 
10571   ins_encode %{
10572     __ cselw(as_Register($dst$$reg),
10573              as_Register($src$$reg),
10574              zr,
10575              (Assembler::Condition)$cmp$$cmpcode);
10576   %}
10577 
10578   ins_pipe(icond_reg);
10579 %}
10580 
10581 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10582 %{
10583   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10584 
10585   ins_cost(INSN_COST * 3);
10586 
10587   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10588   ins_encode %{
10589     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10590     __ fcsels(as_FloatRegister($dst$$reg),
10591               as_FloatRegister($src2$$reg),
10592               as_FloatRegister($src1$$reg),
10593               cond);
10594   %}
10595 
10596   ins_pipe(fp_cond_reg_reg_s);
10597 %}
10598 
10599 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10600 %{
10601   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10602 
10603   ins_cost(INSN_COST * 3);
10604 
10605   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10606   ins_encode %{
10607     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10608     __ fcsels(as_FloatRegister($dst$$reg),
10609               as_FloatRegister($src2$$reg),
10610               as_FloatRegister($src1$$reg),
10611               cond);
10612   %}
10613 
10614   ins_pipe(fp_cond_reg_reg_s);
10615 %}
10616 
10617 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10618 %{
10619   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10620 
10621   ins_cost(INSN_COST * 3);
10622 
10623   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10624   ins_encode %{
10625     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10626     __ fcseld(as_FloatRegister($dst$$reg),
10627               as_FloatRegister($src2$$reg),
10628               as_FloatRegister($src1$$reg),
10629               cond);
10630   %}
10631 
10632   ins_pipe(fp_cond_reg_reg_d);
10633 %}
10634 
10635 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10636 %{
10637   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10638 
10639   ins_cost(INSN_COST * 3);
10640 
10641   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10642   ins_encode %{
10643     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10644     __ fcseld(as_FloatRegister($dst$$reg),
10645               as_FloatRegister($src2$$reg),
10646               as_FloatRegister($src1$$reg),
10647               cond);
10648   %}
10649 
10650   ins_pipe(fp_cond_reg_reg_d);
10651 %}
10652 
10653 // ============================================================================
10654 // Arithmetic Instructions
10655 //
10656 
10657 // Integer Addition
10658 
10659 // TODO
10660 // these currently employ operations which do not set CR and hence are
10661 // not flagged as killing CR but we would like to isolate the cases
10662 // where we want to set flags from those where we don't. need to work
10663 // out how to do that.
10664 
10665 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10666   match(Set dst (AddI src1 src2));
10667 
10668   ins_cost(INSN_COST);
10669   format %{ "addw  $dst, $src1, $src2" %}
10670 
10671   ins_encode %{
10672     __ addw(as_Register($dst$$reg),
10673             as_Register($src1$$reg),
10674             as_Register($src2$$reg));
10675   %}
10676 
10677   ins_pipe(ialu_reg_reg);
10678 %}
10679 
10680 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10681   match(Set dst (AddI src1 src2));
10682 
10683   ins_cost(INSN_COST);
10684   format %{ "addw $dst, $src1, $src2" %}
10685 
10686   // use opcode to indicate that this is an add not a sub
10687   opcode(0x0);
10688 
10689   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10690 
10691   ins_pipe(ialu_reg_imm);
10692 %}
10693 
10694 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10695   match(Set dst (AddI (ConvL2I src1) src2));
10696 
10697   ins_cost(INSN_COST);
10698   format %{ "addw $dst, $src1, $src2" %}
10699 
10700   // use opcode to indicate that this is an add not a sub
10701   opcode(0x0);
10702 
10703   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10704 
10705   ins_pipe(ialu_reg_imm);
10706 %}
10707 
10708 // Pointer Addition
10709 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10710   match(Set dst (AddP src1 src2));
10711 
10712   ins_cost(INSN_COST);
10713   format %{ "add $dst, $src1, $src2\t# ptr" %}
10714 
10715   ins_encode %{
10716     __ add(as_Register($dst$$reg),
10717            as_Register($src1$$reg),
10718            as_Register($src2$$reg));
10719   %}
10720 
10721   ins_pipe(ialu_reg_reg);
10722 %}
10723 
10724 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10725   match(Set dst (AddP src1 (ConvI2L src2)));
10726 
10727   ins_cost(1.9 * INSN_COST);
10728   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10729 
10730   ins_encode %{
10731     __ add(as_Register($dst$$reg),
10732            as_Register($src1$$reg),
10733            as_Register($src2$$reg), ext::sxtw);
10734   %}
10735 
10736   ins_pipe(ialu_reg_reg);
10737 %}
10738 
10739 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10740   match(Set dst (AddP src1 (LShiftL src2 scale)));
10741 
10742   ins_cost(1.9 * INSN_COST);
10743   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10744 
10745   ins_encode %{
10746     __ lea(as_Register($dst$$reg),
10747            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10748                    Address::lsl($scale$$constant)));
10749   %}
10750 
10751   ins_pipe(ialu_reg_reg_shift);
10752 %}
10753 
10754 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10755   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10756 
10757   ins_cost(1.9 * INSN_COST);
10758   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10759 
10760   ins_encode %{
10761     __ lea(as_Register($dst$$reg),
10762            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10763                    Address::sxtw($scale$$constant)));
10764   %}
10765 
10766   ins_pipe(ialu_reg_reg_shift);
10767 %}
10768 
10769 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10770   match(Set dst (LShiftL (ConvI2L src) scale));
10771 
10772   ins_cost(INSN_COST);
10773   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10774 
10775   ins_encode %{
10776     __ sbfiz(as_Register($dst$$reg),
10777           as_Register($src$$reg),
10778           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10779   %}
10780 
10781   ins_pipe(ialu_reg_shift);
10782 %}
10783 
10784 // Pointer Immediate Addition
10785 // n.b. this needs to be more expensive than using an indirect memory
10786 // operand
10787 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10788   match(Set dst (AddP src1 src2));
10789 
10790   ins_cost(INSN_COST);
10791   format %{ "add $dst, $src1, $src2\t# ptr" %}
10792 
10793   // use opcode to indicate that this is an add not a sub
10794   opcode(0x0);
10795 
10796   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10797 
10798   ins_pipe(ialu_reg_imm);
10799 %}
10800 
10801 // Long Addition
10802 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10803 
10804   match(Set dst (AddL src1 src2));
10805 
10806   ins_cost(INSN_COST);
10807   format %{ "add  $dst, $src1, $src2" %}
10808 
10809   ins_encode %{
10810     __ add(as_Register($dst$$reg),
10811            as_Register($src1$$reg),
10812            as_Register($src2$$reg));
10813   %}
10814 
10815   ins_pipe(ialu_reg_reg);
10816 %}
10817 
10818 // No constant pool entries requiredLong Immediate Addition.
10819 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10820   match(Set dst (AddL src1 src2));
10821 
10822   ins_cost(INSN_COST);
10823   format %{ "add $dst, $src1, $src2" %}
10824 
10825   // use opcode to indicate that this is an add not a sub
10826   opcode(0x0);
10827 
10828   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10829 
10830   ins_pipe(ialu_reg_imm);
10831 %}
10832 
10833 // Integer Subtraction
10834 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10835   match(Set dst (SubI src1 src2));
10836 
10837   ins_cost(INSN_COST);
10838   format %{ "subw  $dst, $src1, $src2" %}
10839 
10840   ins_encode %{
10841     __ subw(as_Register($dst$$reg),
10842             as_Register($src1$$reg),
10843             as_Register($src2$$reg));
10844   %}
10845 
10846   ins_pipe(ialu_reg_reg);
10847 %}
10848 
10849 // Immediate Subtraction
10850 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10851   match(Set dst (SubI src1 src2));
10852 
10853   ins_cost(INSN_COST);
10854   format %{ "subw $dst, $src1, $src2" %}
10855 
10856   // use opcode to indicate that this is a sub not an add
10857   opcode(0x1);
10858 
10859   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10860 
10861   ins_pipe(ialu_reg_imm);
10862 %}
10863 
10864 // Long Subtraction
10865 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10866 
10867   match(Set dst (SubL src1 src2));
10868 
10869   ins_cost(INSN_COST);
10870   format %{ "sub  $dst, $src1, $src2" %}
10871 
10872   ins_encode %{
10873     __ sub(as_Register($dst$$reg),
10874            as_Register($src1$$reg),
10875            as_Register($src2$$reg));
10876   %}
10877 
10878   ins_pipe(ialu_reg_reg);
10879 %}
10880 
10881 // No constant pool entries requiredLong Immediate Subtraction.
10882 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10883   match(Set dst (SubL src1 src2));
10884 
10885   ins_cost(INSN_COST);
10886   format %{ "sub$dst, $src1, $src2" %}
10887 
10888   // use opcode to indicate that this is a sub not an add
10889   opcode(0x1);
10890 
10891   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10892 
10893   ins_pipe(ialu_reg_imm);
10894 %}
10895 
10896 // Integer Negation (special case for sub)
10897 
10898 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10899   match(Set dst (SubI zero src));
10900 
10901   ins_cost(INSN_COST);
10902   format %{ "negw $dst, $src\t# int" %}
10903 
10904   ins_encode %{
10905     __ negw(as_Register($dst$$reg),
10906             as_Register($src$$reg));
10907   %}
10908 
10909   ins_pipe(ialu_reg);
10910 %}
10911 
10912 // Long Negation
10913 
10914 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10915   match(Set dst (SubL zero src));
10916 
10917   ins_cost(INSN_COST);
10918   format %{ "neg $dst, $src\t# long" %}
10919 
10920   ins_encode %{
10921     __ neg(as_Register($dst$$reg),
10922            as_Register($src$$reg));
10923   %}
10924 
10925   ins_pipe(ialu_reg);
10926 %}
10927 
10928 // Integer Multiply
10929 
10930 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10931   match(Set dst (MulI src1 src2));
10932 
10933   ins_cost(INSN_COST * 3);
10934   format %{ "mulw  $dst, $src1, $src2" %}
10935 
10936   ins_encode %{
10937     __ mulw(as_Register($dst$$reg),
10938             as_Register($src1$$reg),
10939             as_Register($src2$$reg));
10940   %}
10941 
10942   ins_pipe(imul_reg_reg);
10943 %}
10944 
10945 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10946   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10947 
10948   ins_cost(INSN_COST * 3);
10949   format %{ "smull  $dst, $src1, $src2" %}
10950 
10951   ins_encode %{
10952     __ smull(as_Register($dst$$reg),
10953              as_Register($src1$$reg),
10954              as_Register($src2$$reg));
10955   %}
10956 
10957   ins_pipe(imul_reg_reg);
10958 %}
10959 
10960 // Long Multiply
10961 
10962 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10963   match(Set dst (MulL src1 src2));
10964 
10965   ins_cost(INSN_COST * 5);
10966   format %{ "mul  $dst, $src1, $src2" %}
10967 
10968   ins_encode %{
10969     __ mul(as_Register($dst$$reg),
10970            as_Register($src1$$reg),
10971            as_Register($src2$$reg));
10972   %}
10973 
10974   ins_pipe(lmul_reg_reg);
10975 %}
10976 
10977 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10978 %{
10979   match(Set dst (MulHiL src1 src2));
10980 
10981   ins_cost(INSN_COST * 7);
10982   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10983 
10984   ins_encode %{
10985     __ smulh(as_Register($dst$$reg),
10986              as_Register($src1$$reg),
10987              as_Register($src2$$reg));
10988   %}
10989 
10990   ins_pipe(lmul_reg_reg);
10991 %}
10992 
10993 // Combined Integer Multiply & Add/Sub
10994 
10995 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10996   match(Set dst (AddI src3 (MulI src1 src2)));
10997 
10998   ins_cost(INSN_COST * 3);
10999   format %{ "madd  $dst, $src1, $src2, $src3" %}
11000 
11001   ins_encode %{
11002     __ maddw(as_Register($dst$$reg),
11003              as_Register($src1$$reg),
11004              as_Register($src2$$reg),
11005              as_Register($src3$$reg));
11006   %}
11007 
11008   ins_pipe(imac_reg_reg);
11009 %}
11010 
11011 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11012   match(Set dst (SubI src3 (MulI src1 src2)));
11013 
11014   ins_cost(INSN_COST * 3);
11015   format %{ "msub  $dst, $src1, $src2, $src3" %}
11016 
11017   ins_encode %{
11018     __ msubw(as_Register($dst$$reg),
11019              as_Register($src1$$reg),
11020              as_Register($src2$$reg),
11021              as_Register($src3$$reg));
11022   %}
11023 
11024   ins_pipe(imac_reg_reg);
11025 %}
11026 
11027 // Combined Integer Multiply & Neg
11028 
11029 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11030   match(Set dst (MulI (SubI zero src1) src2));
11031   match(Set dst (MulI src1 (SubI zero src2)));
11032 
11033   ins_cost(INSN_COST * 3);
11034   format %{ "mneg  $dst, $src1, $src2" %}
11035 
11036   ins_encode %{
11037     __ mnegw(as_Register($dst$$reg),
11038              as_Register($src1$$reg),
11039              as_Register($src2$$reg));
11040   %}
11041 
11042   ins_pipe(imac_reg_reg);
11043 %}
11044 
11045 // Combined Long Multiply & Add/Sub
11046 
11047 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11048   match(Set dst (AddL src3 (MulL src1 src2)));
11049 
11050   ins_cost(INSN_COST * 5);
11051   format %{ "madd  $dst, $src1, $src2, $src3" %}
11052 
11053   ins_encode %{
11054     __ madd(as_Register($dst$$reg),
11055             as_Register($src1$$reg),
11056             as_Register($src2$$reg),
11057             as_Register($src3$$reg));
11058   %}
11059 
11060   ins_pipe(lmac_reg_reg);
11061 %}
11062 
11063 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11064   match(Set dst (SubL src3 (MulL src1 src2)));
11065 
11066   ins_cost(INSN_COST * 5);
11067   format %{ "msub  $dst, $src1, $src2, $src3" %}
11068 
11069   ins_encode %{
11070     __ msub(as_Register($dst$$reg),
11071             as_Register($src1$$reg),
11072             as_Register($src2$$reg),
11073             as_Register($src3$$reg));
11074   %}
11075 
11076   ins_pipe(lmac_reg_reg);
11077 %}
11078 
11079 // Combined Long Multiply & Neg
11080 
11081 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11082   match(Set dst (MulL (SubL zero src1) src2));
11083   match(Set dst (MulL src1 (SubL zero src2)));
11084 
11085   ins_cost(INSN_COST * 5);
11086   format %{ "mneg  $dst, $src1, $src2" %}
11087 
11088   ins_encode %{
11089     __ mneg(as_Register($dst$$reg),
11090             as_Register($src1$$reg),
11091             as_Register($src2$$reg));
11092   %}
11093 
11094   ins_pipe(lmac_reg_reg);
11095 %}
11096 
11097 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11098 
11099 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11100   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11101 
11102   ins_cost(INSN_COST * 3);
11103   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11104 
11105   ins_encode %{
11106     __ smaddl(as_Register($dst$$reg),
11107               as_Register($src1$$reg),
11108               as_Register($src2$$reg),
11109               as_Register($src3$$reg));
11110   %}
11111 
11112   ins_pipe(imac_reg_reg);
11113 %}
11114 
11115 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11116   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11117 
11118   ins_cost(INSN_COST * 3);
11119   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11120 
11121   ins_encode %{
11122     __ smsubl(as_Register($dst$$reg),
11123               as_Register($src1$$reg),
11124               as_Register($src2$$reg),
11125               as_Register($src3$$reg));
11126   %}
11127 
11128   ins_pipe(imac_reg_reg);
11129 %}
11130 
11131 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11132   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11133   match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
11134 
11135   ins_cost(INSN_COST * 3);
11136   format %{ "smnegl  $dst, $src1, $src2" %}
11137 
11138   ins_encode %{
11139     __ smnegl(as_Register($dst$$reg),
11140               as_Register($src1$$reg),
11141               as_Register($src2$$reg));
11142   %}
11143 
11144   ins_pipe(imac_reg_reg);
11145 %}
11146 
11147 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11148 
11149 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11150   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11151 
11152   ins_cost(INSN_COST * 5);
11153   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11154             "maddw $dst, $src3, $src4, rscratch1" %}
11155 
11156   ins_encode %{
11157     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11158     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11159 
11160   ins_pipe(imac_reg_reg);
11161 %}
11162 
11163 // Integer Divide
11164 
11165 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11166   match(Set dst (DivI src1 src2));
11167 
11168   ins_cost(INSN_COST * 19);
11169   format %{ "sdivw  $dst, $src1, $src2" %}
11170 
11171   ins_encode(aarch64_enc_divw(dst, src1, src2));
11172   ins_pipe(idiv_reg_reg);
11173 %}
11174 
11175 // Long Divide
11176 
11177 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11178   match(Set dst (DivL src1 src2));
11179 
11180   ins_cost(INSN_COST * 35);
11181   format %{ "sdiv   $dst, $src1, $src2" %}
11182 
11183   ins_encode(aarch64_enc_div(dst, src1, src2));
11184   ins_pipe(ldiv_reg_reg);
11185 %}
11186 
11187 // Integer Remainder
11188 
11189 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11190   match(Set dst (ModI src1 src2));
11191 
11192   ins_cost(INSN_COST * 22);
11193   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11194             "msubw($dst, rscratch1, $src2, $src1" %}
11195 
11196   ins_encode(aarch64_enc_modw(dst, src1, src2));
11197   ins_pipe(idiv_reg_reg);
11198 %}
11199 
11200 // Long Remainder
11201 
11202 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11203   match(Set dst (ModL src1 src2));
11204 
11205   ins_cost(INSN_COST * 38);
11206   format %{ "sdiv   rscratch1, $src1, $src2\n"
11207             "msub($dst, rscratch1, $src2, $src1" %}
11208 
11209   ins_encode(aarch64_enc_mod(dst, src1, src2));
11210   ins_pipe(ldiv_reg_reg);
11211 %}
11212 
11213 // Integer Shifts
11214 
11215 // Shift Left Register
11216 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11217   match(Set dst (LShiftI src1 src2));
11218 
11219   ins_cost(INSN_COST * 2);
11220   format %{ "lslvw  $dst, $src1, $src2" %}
11221 
11222   ins_encode %{
11223     __ lslvw(as_Register($dst$$reg),
11224              as_Register($src1$$reg),
11225              as_Register($src2$$reg));
11226   %}
11227 
11228   ins_pipe(ialu_reg_reg_vshift);
11229 %}
11230 
11231 // Shift Left Immediate
11232 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11233   match(Set dst (LShiftI src1 src2));
11234 
11235   ins_cost(INSN_COST);
11236   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11237 
11238   ins_encode %{
11239     __ lslw(as_Register($dst$$reg),
11240             as_Register($src1$$reg),
11241             $src2$$constant & 0x1f);
11242   %}
11243 
11244   ins_pipe(ialu_reg_shift);
11245 %}
11246 
11247 // Shift Right Logical Register
11248 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11249   match(Set dst (URShiftI src1 src2));
11250 
11251   ins_cost(INSN_COST * 2);
11252   format %{ "lsrvw  $dst, $src1, $src2" %}
11253 
11254   ins_encode %{
11255     __ lsrvw(as_Register($dst$$reg),
11256              as_Register($src1$$reg),
11257              as_Register($src2$$reg));
11258   %}
11259 
11260   ins_pipe(ialu_reg_reg_vshift);
11261 %}
11262 
11263 // Shift Right Logical Immediate
11264 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11265   match(Set dst (URShiftI src1 src2));
11266 
11267   ins_cost(INSN_COST);
11268   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11269 
11270   ins_encode %{
11271     __ lsrw(as_Register($dst$$reg),
11272             as_Register($src1$$reg),
11273             $src2$$constant & 0x1f);
11274   %}
11275 
11276   ins_pipe(ialu_reg_shift);
11277 %}
11278 
11279 // Shift Right Arithmetic Register
11280 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11281   match(Set dst (RShiftI src1 src2));
11282 
11283   ins_cost(INSN_COST * 2);
11284   format %{ "asrvw  $dst, $src1, $src2" %}
11285 
11286   ins_encode %{
11287     __ asrvw(as_Register($dst$$reg),
11288              as_Register($src1$$reg),
11289              as_Register($src2$$reg));
11290   %}
11291 
11292   ins_pipe(ialu_reg_reg_vshift);
11293 %}
11294 
11295 // Shift Right Arithmetic Immediate
11296 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11297   match(Set dst (RShiftI src1 src2));
11298 
11299   ins_cost(INSN_COST);
11300   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11301 
11302   ins_encode %{
11303     __ asrw(as_Register($dst$$reg),
11304             as_Register($src1$$reg),
11305             $src2$$constant & 0x1f);
11306   %}
11307 
11308   ins_pipe(ialu_reg_shift);
11309 %}
11310 
11311 // Combined Int Mask and Right Shift (using UBFM)
11312 // TODO
11313 
11314 // Long Shifts
11315 
11316 // Shift Left Register
11317 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11318   match(Set dst (LShiftL src1 src2));
11319 
11320   ins_cost(INSN_COST * 2);
11321   format %{ "lslv  $dst, $src1, $src2" %}
11322 
11323   ins_encode %{
11324     __ lslv(as_Register($dst$$reg),
11325             as_Register($src1$$reg),
11326             as_Register($src2$$reg));
11327   %}
11328 
11329   ins_pipe(ialu_reg_reg_vshift);
11330 %}
11331 
11332 // Shift Left Immediate
11333 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11334   match(Set dst (LShiftL src1 src2));
11335 
11336   ins_cost(INSN_COST);
11337   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11338 
11339   ins_encode %{
11340     __ lsl(as_Register($dst$$reg),
11341             as_Register($src1$$reg),
11342             $src2$$constant & 0x3f);
11343   %}
11344 
11345   ins_pipe(ialu_reg_shift);
11346 %}
11347 
11348 // Shift Right Logical Register
11349 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11350   match(Set dst (URShiftL src1 src2));
11351 
11352   ins_cost(INSN_COST * 2);
11353   format %{ "lsrv  $dst, $src1, $src2" %}
11354 
11355   ins_encode %{
11356     __ lsrv(as_Register($dst$$reg),
11357             as_Register($src1$$reg),
11358             as_Register($src2$$reg));
11359   %}
11360 
11361   ins_pipe(ialu_reg_reg_vshift);
11362 %}
11363 
11364 // Shift Right Logical Immediate
11365 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11366   match(Set dst (URShiftL src1 src2));
11367 
11368   ins_cost(INSN_COST);
11369   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11370 
11371   ins_encode %{
11372     __ lsr(as_Register($dst$$reg),
11373            as_Register($src1$$reg),
11374            $src2$$constant & 0x3f);
11375   %}
11376 
11377   ins_pipe(ialu_reg_shift);
11378 %}
11379 
11380 // A special-case pattern for card table stores.
11381 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11382   match(Set dst (URShiftL (CastP2X src1) src2));
11383 
11384   ins_cost(INSN_COST);
11385   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11386 
11387   ins_encode %{
11388     __ lsr(as_Register($dst$$reg),
11389            as_Register($src1$$reg),
11390            $src2$$constant & 0x3f);
11391   %}
11392 
11393   ins_pipe(ialu_reg_shift);
11394 %}
11395 
11396 // Shift Right Arithmetic Register
11397 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11398   match(Set dst (RShiftL src1 src2));
11399 
11400   ins_cost(INSN_COST * 2);
11401   format %{ "asrv  $dst, $src1, $src2" %}
11402 
11403   ins_encode %{
11404     __ asrv(as_Register($dst$$reg),
11405             as_Register($src1$$reg),
11406             as_Register($src2$$reg));
11407   %}
11408 
11409   ins_pipe(ialu_reg_reg_vshift);
11410 %}
11411 
11412 // Shift Right Arithmetic Immediate
11413 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11414   match(Set dst (RShiftL src1 src2));
11415 
11416   ins_cost(INSN_COST);
11417   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11418 
11419   ins_encode %{
11420     __ asr(as_Register($dst$$reg),
11421            as_Register($src1$$reg),
11422            $src2$$constant & 0x3f);
11423   %}
11424 
11425   ins_pipe(ialu_reg_shift);
11426 %}
11427 
11428 // BEGIN This section of the file is automatically generated. Do not edit --------------
11429 // This section is generated from aarch64_ad.m4
11430 
11431 // This pattern is automatically generated from aarch64_ad.m4
11432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11433 instruct regL_not_reg(iRegLNoSp dst,
11434                          iRegL src1, immL_M1 m1,
11435                          rFlagsReg cr) %{
11436   match(Set dst (XorL src1 m1));
11437   ins_cost(INSN_COST);
11438   format %{ "eon  $dst, $src1, zr" %}
11439 
11440   ins_encode %{
11441     __ eon(as_Register($dst$$reg),
11442               as_Register($src1$$reg),
11443               zr,
11444               Assembler::LSL, 0);
11445   %}
11446 
11447   ins_pipe(ialu_reg);
11448 %}
11449 
11450 // This pattern is automatically generated from aarch64_ad.m4
11451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11452 instruct regI_not_reg(iRegINoSp dst,
11453                          iRegIorL2I src1, immI_M1 m1,
11454                          rFlagsReg cr) %{
11455   match(Set dst (XorI src1 m1));
11456   ins_cost(INSN_COST);
11457   format %{ "eonw  $dst, $src1, zr" %}
11458 
11459   ins_encode %{
11460     __ eonw(as_Register($dst$$reg),
11461               as_Register($src1$$reg),
11462               zr,
11463               Assembler::LSL, 0);
11464   %}
11465 
11466   ins_pipe(ialu_reg);
11467 %}
11468 
11469 // This pattern is automatically generated from aarch64_ad.m4
11470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11471 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11472                               immI0 zero, iRegIorL2I src1, immI src2) %{
11473   match(Set dst (SubI zero (URShiftI src1 src2)));
11474 
11475   ins_cost(1.9 * INSN_COST);
11476   format %{ "negw  $dst, $src1, LSR $src2" %}
11477 
11478   ins_encode %{
11479     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11480             Assembler::LSR, $src2$$constant & 0x1f);
11481   %}
11482 
11483   ins_pipe(ialu_reg_shift);
11484 %}
11485 
11486 // This pattern is automatically generated from aarch64_ad.m4
11487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11488 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11489                               immI0 zero, iRegIorL2I src1, immI src2) %{
11490   match(Set dst (SubI zero (RShiftI src1 src2)));
11491 
11492   ins_cost(1.9 * INSN_COST);
11493   format %{ "negw  $dst, $src1, ASR $src2" %}
11494 
11495   ins_encode %{
11496     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11497             Assembler::ASR, $src2$$constant & 0x1f);
11498   %}
11499 
11500   ins_pipe(ialu_reg_shift);
11501 %}
11502 
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11506                               immI0 zero, iRegIorL2I src1, immI src2) %{
11507   match(Set dst (SubI zero (LShiftI src1 src2)));
11508 
11509   ins_cost(1.9 * INSN_COST);
11510   format %{ "negw  $dst, $src1, LSL $src2" %}
11511 
11512   ins_encode %{
11513     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11514             Assembler::LSL, $src2$$constant & 0x1f);
11515   %}
11516 
11517   ins_pipe(ialu_reg_shift);
11518 %}
11519 
11520 // This pattern is automatically generated from aarch64_ad.m4
11521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11522 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11523                               immL0 zero, iRegL src1, immI src2) %{
11524   match(Set dst (SubL zero (URShiftL src1 src2)));
11525 
11526   ins_cost(1.9 * INSN_COST);
11527   format %{ "neg  $dst, $src1, LSR $src2" %}
11528 
11529   ins_encode %{
11530     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11531             Assembler::LSR, $src2$$constant & 0x3f);
11532   %}
11533 
11534   ins_pipe(ialu_reg_shift);
11535 %}
11536 
11537 // This pattern is automatically generated from aarch64_ad.m4
11538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11539 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11540                               immL0 zero, iRegL src1, immI src2) %{
11541   match(Set dst (SubL zero (RShiftL src1 src2)));
11542 
11543   ins_cost(1.9 * INSN_COST);
11544   format %{ "neg  $dst, $src1, ASR $src2" %}
11545 
11546   ins_encode %{
11547     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11548             Assembler::ASR, $src2$$constant & 0x3f);
11549   %}
11550 
11551   ins_pipe(ialu_reg_shift);
11552 %}
11553 
11554 // This pattern is automatically generated from aarch64_ad.m4
11555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11556 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11557                               immL0 zero, iRegL src1, immI src2) %{
11558   match(Set dst (SubL zero (LShiftL src1 src2)));
11559 
11560   ins_cost(1.9 * INSN_COST);
11561   format %{ "neg  $dst, $src1, LSL $src2" %}
11562 
11563   ins_encode %{
11564     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11565             Assembler::LSL, $src2$$constant & 0x3f);
11566   %}
11567 
11568   ins_pipe(ialu_reg_shift);
11569 %}
11570 
11571 // This pattern is automatically generated from aarch64_ad.m4
11572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11573 instruct AndI_reg_not_reg(iRegINoSp dst,
11574                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11575   match(Set dst (AndI src1 (XorI src2 m1)));
11576   ins_cost(INSN_COST);
11577   format %{ "bicw  $dst, $src1, $src2" %}
11578 
11579   ins_encode %{
11580     __ bicw(as_Register($dst$$reg),
11581               as_Register($src1$$reg),
11582               as_Register($src2$$reg),
11583               Assembler::LSL, 0);
11584   %}
11585 
11586   ins_pipe(ialu_reg_reg);
11587 %}
11588 
11589 // This pattern is automatically generated from aarch64_ad.m4
11590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11591 instruct AndL_reg_not_reg(iRegLNoSp dst,
11592                          iRegL src1, iRegL src2, immL_M1 m1) %{
11593   match(Set dst (AndL src1 (XorL src2 m1)));
11594   ins_cost(INSN_COST);
11595   format %{ "bic  $dst, $src1, $src2" %}
11596 
11597   ins_encode %{
11598     __ bic(as_Register($dst$$reg),
11599               as_Register($src1$$reg),
11600               as_Register($src2$$reg),
11601               Assembler::LSL, 0);
11602   %}
11603 
11604   ins_pipe(ialu_reg_reg);
11605 %}
11606 
11607 // This pattern is automatically generated from aarch64_ad.m4
11608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11609 instruct OrI_reg_not_reg(iRegINoSp dst,
11610                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11611   match(Set dst (OrI src1 (XorI src2 m1)));
11612   ins_cost(INSN_COST);
11613   format %{ "ornw  $dst, $src1, $src2" %}
11614 
11615   ins_encode %{
11616     __ ornw(as_Register($dst$$reg),
11617               as_Register($src1$$reg),
11618               as_Register($src2$$reg),
11619               Assembler::LSL, 0);
11620   %}
11621 
11622   ins_pipe(ialu_reg_reg);
11623 %}
11624 
11625 // This pattern is automatically generated from aarch64_ad.m4
11626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11627 instruct OrL_reg_not_reg(iRegLNoSp dst,
11628                          iRegL src1, iRegL src2, immL_M1 m1) %{
11629   match(Set dst (OrL src1 (XorL src2 m1)));
11630   ins_cost(INSN_COST);
11631   format %{ "orn  $dst, $src1, $src2" %}
11632 
11633   ins_encode %{
11634     __ orn(as_Register($dst$$reg),
11635               as_Register($src1$$reg),
11636               as_Register($src2$$reg),
11637               Assembler::LSL, 0);
11638   %}
11639 
11640   ins_pipe(ialu_reg_reg);
11641 %}
11642 
11643 // This pattern is automatically generated from aarch64_ad.m4
11644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11645 instruct XorI_reg_not_reg(iRegINoSp dst,
11646                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11647   match(Set dst (XorI m1 (XorI src2 src1)));
11648   ins_cost(INSN_COST);
11649   format %{ "eonw  $dst, $src1, $src2" %}
11650 
11651   ins_encode %{
11652     __ eonw(as_Register($dst$$reg),
11653               as_Register($src1$$reg),
11654               as_Register($src2$$reg),
11655               Assembler::LSL, 0);
11656   %}
11657 
11658   ins_pipe(ialu_reg_reg);
11659 %}
11660 
11661 // This pattern is automatically generated from aarch64_ad.m4
11662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11663 instruct XorL_reg_not_reg(iRegLNoSp dst,
11664                          iRegL src1, iRegL src2, immL_M1 m1) %{
11665   match(Set dst (XorL m1 (XorL src2 src1)));
11666   ins_cost(INSN_COST);
11667   format %{ "eon  $dst, $src1, $src2" %}
11668 
11669   ins_encode %{
11670     __ eon(as_Register($dst$$reg),
11671               as_Register($src1$$reg),
11672               as_Register($src2$$reg),
11673               Assembler::LSL, 0);
11674   %}
11675 
11676   ins_pipe(ialu_reg_reg);
11677 %}
11678 
11679 // This pattern is automatically generated from aarch64_ad.m4
11680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11681 // val & (-1 ^ (val >>> shift)) ==> bicw
11682 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11683                          iRegIorL2I src1, iRegIorL2I src2,
11684                          immI src3, immI_M1 src4) %{
11685   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11686   ins_cost(1.9 * INSN_COST);
11687   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11688 
11689   ins_encode %{
11690     __ bicw(as_Register($dst$$reg),
11691               as_Register($src1$$reg),
11692               as_Register($src2$$reg),
11693               Assembler::LSR,
11694               $src3$$constant & 0x1f);
11695   %}
11696 
11697   ins_pipe(ialu_reg_reg_shift);
11698 %}
11699 
11700 // This pattern is automatically generated from aarch64_ad.m4
11701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11702 // val & (-1 ^ (val >>> shift)) ==> bic
11703 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11704                          iRegL src1, iRegL src2,
11705                          immI src3, immL_M1 src4) %{
11706   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11707   ins_cost(1.9 * INSN_COST);
11708   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11709 
11710   ins_encode %{
11711     __ bic(as_Register($dst$$reg),
11712               as_Register($src1$$reg),
11713               as_Register($src2$$reg),
11714               Assembler::LSR,
11715               $src3$$constant & 0x3f);
11716   %}
11717 
11718   ins_pipe(ialu_reg_reg_shift);
11719 %}
11720 
11721 // This pattern is automatically generated from aarch64_ad.m4
11722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11723 // val & (-1 ^ (val >> shift)) ==> bicw
11724 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11725                          iRegIorL2I src1, iRegIorL2I src2,
11726                          immI src3, immI_M1 src4) %{
11727   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11728   ins_cost(1.9 * INSN_COST);
11729   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11730 
11731   ins_encode %{
11732     __ bicw(as_Register($dst$$reg),
11733               as_Register($src1$$reg),
11734               as_Register($src2$$reg),
11735               Assembler::ASR,
11736               $src3$$constant & 0x1f);
11737   %}
11738 
11739   ins_pipe(ialu_reg_reg_shift);
11740 %}
11741 
11742 // This pattern is automatically generated from aarch64_ad.m4
11743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11744 // val & (-1 ^ (val >> shift)) ==> bic
11745 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11746                          iRegL src1, iRegL src2,
11747                          immI src3, immL_M1 src4) %{
11748   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11749   ins_cost(1.9 * INSN_COST);
11750   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11751 
11752   ins_encode %{
11753     __ bic(as_Register($dst$$reg),
11754               as_Register($src1$$reg),
11755               as_Register($src2$$reg),
11756               Assembler::ASR,
11757               $src3$$constant & 0x3f);
11758   %}
11759 
11760   ins_pipe(ialu_reg_reg_shift);
11761 %}
11762 
11763 // This pattern is automatically generated from aarch64_ad.m4
11764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11765 // val & (-1 ^ (val ror shift)) ==> bicw
11766 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11767                          iRegIorL2I src1, iRegIorL2I src2,
11768                          immI src3, immI_M1 src4) %{
11769   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11770   ins_cost(1.9 * INSN_COST);
11771   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11772 
11773   ins_encode %{
11774     __ bicw(as_Register($dst$$reg),
11775               as_Register($src1$$reg),
11776               as_Register($src2$$reg),
11777               Assembler::ROR,
11778               $src3$$constant & 0x1f);
11779   %}
11780 
11781   ins_pipe(ialu_reg_reg_shift);
11782 %}
11783 
11784 // This pattern is automatically generated from aarch64_ad.m4
11785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11786 // val & (-1 ^ (val ror shift)) ==> bic
11787 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11788                          iRegL src1, iRegL src2,
11789                          immI src3, immL_M1 src4) %{
11790   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11791   ins_cost(1.9 * INSN_COST);
11792   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11793 
11794   ins_encode %{
11795     __ bic(as_Register($dst$$reg),
11796               as_Register($src1$$reg),
11797               as_Register($src2$$reg),
11798               Assembler::ROR,
11799               $src3$$constant & 0x3f);
11800   %}
11801 
11802   ins_pipe(ialu_reg_reg_shift);
11803 %}
11804 
11805 // This pattern is automatically generated from aarch64_ad.m4
11806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11807 // val & (-1 ^ (val << shift)) ==> bicw
11808 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11809                          iRegIorL2I src1, iRegIorL2I src2,
11810                          immI src3, immI_M1 src4) %{
11811   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11812   ins_cost(1.9 * INSN_COST);
11813   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11814 
11815   ins_encode %{
11816     __ bicw(as_Register($dst$$reg),
11817               as_Register($src1$$reg),
11818               as_Register($src2$$reg),
11819               Assembler::LSL,
11820               $src3$$constant & 0x1f);
11821   %}
11822 
11823   ins_pipe(ialu_reg_reg_shift);
11824 %}
11825 
11826 // This pattern is automatically generated from aarch64_ad.m4
11827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11828 // val & (-1 ^ (val << shift)) ==> bic
11829 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11830                          iRegL src1, iRegL src2,
11831                          immI src3, immL_M1 src4) %{
11832   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11833   ins_cost(1.9 * INSN_COST);
11834   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11835 
11836   ins_encode %{
11837     __ bic(as_Register($dst$$reg),
11838               as_Register($src1$$reg),
11839               as_Register($src2$$reg),
11840               Assembler::LSL,
11841               $src3$$constant & 0x3f);
11842   %}
11843 
11844   ins_pipe(ialu_reg_reg_shift);
11845 %}
11846 
11847 // This pattern is automatically generated from aarch64_ad.m4
11848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11849 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11850 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11851                          iRegIorL2I src1, iRegIorL2I src2,
11852                          immI src3, immI_M1 src4) %{
11853   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11854   ins_cost(1.9 * INSN_COST);
11855   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11856 
11857   ins_encode %{
11858     __ eonw(as_Register($dst$$reg),
11859               as_Register($src1$$reg),
11860               as_Register($src2$$reg),
11861               Assembler::LSR,
11862               $src3$$constant & 0x1f);
11863   %}
11864 
11865   ins_pipe(ialu_reg_reg_shift);
11866 %}
11867 
11868 // This pattern is automatically generated from aarch64_ad.m4
11869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11870 // val ^ (-1 ^ (val >>> shift)) ==> eon
11871 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11872                          iRegL src1, iRegL src2,
11873                          immI src3, immL_M1 src4) %{
11874   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11875   ins_cost(1.9 * INSN_COST);
11876   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11877 
11878   ins_encode %{
11879     __ eon(as_Register($dst$$reg),
11880               as_Register($src1$$reg),
11881               as_Register($src2$$reg),
11882               Assembler::LSR,
11883               $src3$$constant & 0x3f);
11884   %}
11885 
11886   ins_pipe(ialu_reg_reg_shift);
11887 %}
11888 
11889 // This pattern is automatically generated from aarch64_ad.m4
11890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11891 // val ^ (-1 ^ (val >> shift)) ==> eonw
11892 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11893                          iRegIorL2I src1, iRegIorL2I src2,
11894                          immI src3, immI_M1 src4) %{
11895   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11896   ins_cost(1.9 * INSN_COST);
11897   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11898 
11899   ins_encode %{
11900     __ eonw(as_Register($dst$$reg),
11901               as_Register($src1$$reg),
11902               as_Register($src2$$reg),
11903               Assembler::ASR,
11904               $src3$$constant & 0x1f);
11905   %}
11906 
11907   ins_pipe(ialu_reg_reg_shift);
11908 %}
11909 
11910 // This pattern is automatically generated from aarch64_ad.m4
11911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11912 // val ^ (-1 ^ (val >> shift)) ==> eon
11913 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11914                          iRegL src1, iRegL src2,
11915                          immI src3, immL_M1 src4) %{
11916   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11917   ins_cost(1.9 * INSN_COST);
11918   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11919 
11920   ins_encode %{
11921     __ eon(as_Register($dst$$reg),
11922               as_Register($src1$$reg),
11923               as_Register($src2$$reg),
11924               Assembler::ASR,
11925               $src3$$constant & 0x3f);
11926   %}
11927 
11928   ins_pipe(ialu_reg_reg_shift);
11929 %}
11930 
11931 // This pattern is automatically generated from aarch64_ad.m4
11932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11933 // val ^ (-1 ^ (val ror shift)) ==> eonw
11934 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11935                          iRegIorL2I src1, iRegIorL2I src2,
11936                          immI src3, immI_M1 src4) %{
11937   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11938   ins_cost(1.9 * INSN_COST);
11939   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11940 
11941   ins_encode %{
11942     __ eonw(as_Register($dst$$reg),
11943               as_Register($src1$$reg),
11944               as_Register($src2$$reg),
11945               Assembler::ROR,
11946               $src3$$constant & 0x1f);
11947   %}
11948 
11949   ins_pipe(ialu_reg_reg_shift);
11950 %}
11951 
11952 // This pattern is automatically generated from aarch64_ad.m4
11953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11954 // val ^ (-1 ^ (val ror shift)) ==> eon
11955 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11956                          iRegL src1, iRegL src2,
11957                          immI src3, immL_M1 src4) %{
11958   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11959   ins_cost(1.9 * INSN_COST);
11960   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11961 
11962   ins_encode %{
11963     __ eon(as_Register($dst$$reg),
11964               as_Register($src1$$reg),
11965               as_Register($src2$$reg),
11966               Assembler::ROR,
11967               $src3$$constant & 0x3f);
11968   %}
11969 
11970   ins_pipe(ialu_reg_reg_shift);
11971 %}
11972 
11973 // This pattern is automatically generated from aarch64_ad.m4
11974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11975 // val ^ (-1 ^ (val << shift)) ==> eonw
11976 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11977                          iRegIorL2I src1, iRegIorL2I src2,
11978                          immI src3, immI_M1 src4) %{
11979   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11980   ins_cost(1.9 * INSN_COST);
11981   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11982 
11983   ins_encode %{
11984     __ eonw(as_Register($dst$$reg),
11985               as_Register($src1$$reg),
11986               as_Register($src2$$reg),
11987               Assembler::LSL,
11988               $src3$$constant & 0x1f);
11989   %}
11990 
11991   ins_pipe(ialu_reg_reg_shift);
11992 %}
11993 
11994 // This pattern is automatically generated from aarch64_ad.m4
11995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11996 // val ^ (-1 ^ (val << shift)) ==> eon
11997 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11998                          iRegL src1, iRegL src2,
11999                          immI src3, immL_M1 src4) %{
12000   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12001   ins_cost(1.9 * INSN_COST);
12002   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
12003 
12004   ins_encode %{
12005     __ eon(as_Register($dst$$reg),
12006               as_Register($src1$$reg),
12007               as_Register($src2$$reg),
12008               Assembler::LSL,
12009               $src3$$constant & 0x3f);
12010   %}
12011 
12012   ins_pipe(ialu_reg_reg_shift);
12013 %}
12014 
12015 // This pattern is automatically generated from aarch64_ad.m4
12016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12017 // val | (-1 ^ (val >>> shift)) ==> ornw
12018 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12019                          iRegIorL2I src1, iRegIorL2I src2,
12020                          immI src3, immI_M1 src4) %{
12021   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12022   ins_cost(1.9 * INSN_COST);
12023   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
12024 
12025   ins_encode %{
12026     __ ornw(as_Register($dst$$reg),
12027               as_Register($src1$$reg),
12028               as_Register($src2$$reg),
12029               Assembler::LSR,
12030               $src3$$constant & 0x1f);
12031   %}
12032 
12033   ins_pipe(ialu_reg_reg_shift);
12034 %}
12035 
12036 // This pattern is automatically generated from aarch64_ad.m4
12037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12038 // val | (-1 ^ (val >>> shift)) ==> orn
12039 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12040                          iRegL src1, iRegL src2,
12041                          immI src3, immL_M1 src4) %{
12042   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12043   ins_cost(1.9 * INSN_COST);
12044   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
12045 
12046   ins_encode %{
12047     __ orn(as_Register($dst$$reg),
12048               as_Register($src1$$reg),
12049               as_Register($src2$$reg),
12050               Assembler::LSR,
12051               $src3$$constant & 0x3f);
12052   %}
12053 
12054   ins_pipe(ialu_reg_reg_shift);
12055 %}
12056 
12057 // This pattern is automatically generated from aarch64_ad.m4
12058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12059 // val | (-1 ^ (val >> shift)) ==> ornw
12060 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12061                          iRegIorL2I src1, iRegIorL2I src2,
12062                          immI src3, immI_M1 src4) %{
12063   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12064   ins_cost(1.9 * INSN_COST);
12065   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12066 
12067   ins_encode %{
12068     __ ornw(as_Register($dst$$reg),
12069               as_Register($src1$$reg),
12070               as_Register($src2$$reg),
12071               Assembler::ASR,
12072               $src3$$constant & 0x1f);
12073   %}
12074 
12075   ins_pipe(ialu_reg_reg_shift);
12076 %}
12077 
12078 // This pattern is automatically generated from aarch64_ad.m4
12079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12080 // val | (-1 ^ (val >> shift)) ==> orn
12081 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12082                          iRegL src1, iRegL src2,
12083                          immI src3, immL_M1 src4) %{
12084   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12085   ins_cost(1.9 * INSN_COST);
12086   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12087 
12088   ins_encode %{
12089     __ orn(as_Register($dst$$reg),
12090               as_Register($src1$$reg),
12091               as_Register($src2$$reg),
12092               Assembler::ASR,
12093               $src3$$constant & 0x3f);
12094   %}
12095 
12096   ins_pipe(ialu_reg_reg_shift);
12097 %}
12098 
12099 // This pattern is automatically generated from aarch64_ad.m4
12100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12101 // val | (-1 ^ (val ror shift)) ==> ornw
12102 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12103                          iRegIorL2I src1, iRegIorL2I src2,
12104                          immI src3, immI_M1 src4) %{
12105   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12106   ins_cost(1.9 * INSN_COST);
12107   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12108 
12109   ins_encode %{
12110     __ ornw(as_Register($dst$$reg),
12111               as_Register($src1$$reg),
12112               as_Register($src2$$reg),
12113               Assembler::ROR,
12114               $src3$$constant & 0x1f);
12115   %}
12116 
12117   ins_pipe(ialu_reg_reg_shift);
12118 %}
12119 
12120 // This pattern is automatically generated from aarch64_ad.m4
12121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12122 // val | (-1 ^ (val ror shift)) ==> orn
12123 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12124                          iRegL src1, iRegL src2,
12125                          immI src3, immL_M1 src4) %{
12126   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12127   ins_cost(1.9 * INSN_COST);
12128   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12129 
12130   ins_encode %{
12131     __ orn(as_Register($dst$$reg),
12132               as_Register($src1$$reg),
12133               as_Register($src2$$reg),
12134               Assembler::ROR,
12135               $src3$$constant & 0x3f);
12136   %}
12137 
12138   ins_pipe(ialu_reg_reg_shift);
12139 %}
12140 
12141 // This pattern is automatically generated from aarch64_ad.m4
12142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12143 // val | (-1 ^ (val << shift)) ==> ornw
12144 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12145                          iRegIorL2I src1, iRegIorL2I src2,
12146                          immI src3, immI_M1 src4) %{
12147   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12148   ins_cost(1.9 * INSN_COST);
12149   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12150 
12151   ins_encode %{
12152     __ ornw(as_Register($dst$$reg),
12153               as_Register($src1$$reg),
12154               as_Register($src2$$reg),
12155               Assembler::LSL,
12156               $src3$$constant & 0x1f);
12157   %}
12158 
12159   ins_pipe(ialu_reg_reg_shift);
12160 %}
12161 
12162 // This pattern is automatically generated from aarch64_ad.m4
12163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12164 // val | (-1 ^ (val << shift)) ==> orn
12165 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12166                          iRegL src1, iRegL src2,
12167                          immI src3, immL_M1 src4) %{
12168   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12169   ins_cost(1.9 * INSN_COST);
12170   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12171 
12172   ins_encode %{
12173     __ orn(as_Register($dst$$reg),
12174               as_Register($src1$$reg),
12175               as_Register($src2$$reg),
12176               Assembler::LSL,
12177               $src3$$constant & 0x3f);
12178   %}
12179 
12180   ins_pipe(ialu_reg_reg_shift);
12181 %}
12182 
12183 // This pattern is automatically generated from aarch64_ad.m4
12184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12185 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12186                          iRegIorL2I src1, iRegIorL2I src2,
12187                          immI src3) %{
12188   match(Set dst (AndI src1 (URShiftI src2 src3)));
12189 
12190   ins_cost(1.9 * INSN_COST);
12191   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12192 
12193   ins_encode %{
12194     __ andw(as_Register($dst$$reg),
12195               as_Register($src1$$reg),
12196               as_Register($src2$$reg),
12197               Assembler::LSR,
12198               $src3$$constant & 0x1f);
12199   %}
12200 
12201   ins_pipe(ialu_reg_reg_shift);
12202 %}
12203 
12204 // This pattern is automatically generated from aarch64_ad.m4
12205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12206 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12207                          iRegL src1, iRegL src2,
12208                          immI src3) %{
12209   match(Set dst (AndL src1 (URShiftL src2 src3)));
12210 
12211   ins_cost(1.9 * INSN_COST);
12212   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12213 
12214   ins_encode %{
12215     __ andr(as_Register($dst$$reg),
12216               as_Register($src1$$reg),
12217               as_Register($src2$$reg),
12218               Assembler::LSR,
12219               $src3$$constant & 0x3f);
12220   %}
12221 
12222   ins_pipe(ialu_reg_reg_shift);
12223 %}
12224 
12225 // This pattern is automatically generated from aarch64_ad.m4
12226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12227 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12228                          iRegIorL2I src1, iRegIorL2I src2,
12229                          immI src3) %{
12230   match(Set dst (AndI src1 (RShiftI src2 src3)));
12231 
12232   ins_cost(1.9 * INSN_COST);
12233   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12234 
12235   ins_encode %{
12236     __ andw(as_Register($dst$$reg),
12237               as_Register($src1$$reg),
12238               as_Register($src2$$reg),
12239               Assembler::ASR,
12240               $src3$$constant & 0x1f);
12241   %}
12242 
12243   ins_pipe(ialu_reg_reg_shift);
12244 %}
12245 
12246 // This pattern is automatically generated from aarch64_ad.m4
12247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12248 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12249                          iRegL src1, iRegL src2,
12250                          immI src3) %{
12251   match(Set dst (AndL src1 (RShiftL src2 src3)));
12252 
12253   ins_cost(1.9 * INSN_COST);
12254   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12255 
12256   ins_encode %{
12257     __ andr(as_Register($dst$$reg),
12258               as_Register($src1$$reg),
12259               as_Register($src2$$reg),
12260               Assembler::ASR,
12261               $src3$$constant & 0x3f);
12262   %}
12263 
12264   ins_pipe(ialu_reg_reg_shift);
12265 %}
12266 
12267 // This pattern is automatically generated from aarch64_ad.m4
12268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12269 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12270                          iRegIorL2I src1, iRegIorL2I src2,
12271                          immI src3) %{
12272   match(Set dst (AndI src1 (LShiftI src2 src3)));
12273 
12274   ins_cost(1.9 * INSN_COST);
12275   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12276 
12277   ins_encode %{
12278     __ andw(as_Register($dst$$reg),
12279               as_Register($src1$$reg),
12280               as_Register($src2$$reg),
12281               Assembler::LSL,
12282               $src3$$constant & 0x1f);
12283   %}
12284 
12285   ins_pipe(ialu_reg_reg_shift);
12286 %}
12287 
12288 // This pattern is automatically generated from aarch64_ad.m4
12289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12290 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12291                          iRegL src1, iRegL src2,
12292                          immI src3) %{
12293   match(Set dst (AndL src1 (LShiftL src2 src3)));
12294 
12295   ins_cost(1.9 * INSN_COST);
12296   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12297 
12298   ins_encode %{
12299     __ andr(as_Register($dst$$reg),
12300               as_Register($src1$$reg),
12301               as_Register($src2$$reg),
12302               Assembler::LSL,
12303               $src3$$constant & 0x3f);
12304   %}
12305 
12306   ins_pipe(ialu_reg_reg_shift);
12307 %}
12308 
12309 // This pattern is automatically generated from aarch64_ad.m4
12310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12311 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12312                          iRegIorL2I src1, iRegIorL2I src2,
12313                          immI src3) %{
12314   match(Set dst (AndI src1 (RotateRight src2 src3)));
12315 
12316   ins_cost(1.9 * INSN_COST);
12317   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12318 
12319   ins_encode %{
12320     __ andw(as_Register($dst$$reg),
12321               as_Register($src1$$reg),
12322               as_Register($src2$$reg),
12323               Assembler::ROR,
12324               $src3$$constant & 0x1f);
12325   %}
12326 
12327   ins_pipe(ialu_reg_reg_shift);
12328 %}
12329 
12330 // This pattern is automatically generated from aarch64_ad.m4
12331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12332 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12333                          iRegL src1, iRegL src2,
12334                          immI src3) %{
12335   match(Set dst (AndL src1 (RotateRight src2 src3)));
12336 
12337   ins_cost(1.9 * INSN_COST);
12338   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12339 
12340   ins_encode %{
12341     __ andr(as_Register($dst$$reg),
12342               as_Register($src1$$reg),
12343               as_Register($src2$$reg),
12344               Assembler::ROR,
12345               $src3$$constant & 0x3f);
12346   %}
12347 
12348   ins_pipe(ialu_reg_reg_shift);
12349 %}
12350 
12351 // This pattern is automatically generated from aarch64_ad.m4
12352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12353 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12354                          iRegIorL2I src1, iRegIorL2I src2,
12355                          immI src3) %{
12356   match(Set dst (XorI src1 (URShiftI src2 src3)));
12357 
12358   ins_cost(1.9 * INSN_COST);
12359   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12360 
12361   ins_encode %{
12362     __ eorw(as_Register($dst$$reg),
12363               as_Register($src1$$reg),
12364               as_Register($src2$$reg),
12365               Assembler::LSR,
12366               $src3$$constant & 0x1f);
12367   %}
12368 
12369   ins_pipe(ialu_reg_reg_shift);
12370 %}
12371 
12372 // This pattern is automatically generated from aarch64_ad.m4
12373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12374 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12375                          iRegL src1, iRegL src2,
12376                          immI src3) %{
12377   match(Set dst (XorL src1 (URShiftL src2 src3)));
12378 
12379   ins_cost(1.9 * INSN_COST);
12380   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12381 
12382   ins_encode %{
12383     __ eor(as_Register($dst$$reg),
12384               as_Register($src1$$reg),
12385               as_Register($src2$$reg),
12386               Assembler::LSR,
12387               $src3$$constant & 0x3f);
12388   %}
12389 
12390   ins_pipe(ialu_reg_reg_shift);
12391 %}
12392 
12393 // This pattern is automatically generated from aarch64_ad.m4
12394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12395 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12396                          iRegIorL2I src1, iRegIorL2I src2,
12397                          immI src3) %{
12398   match(Set dst (XorI src1 (RShiftI src2 src3)));
12399 
12400   ins_cost(1.9 * INSN_COST);
12401   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12402 
12403   ins_encode %{
12404     __ eorw(as_Register($dst$$reg),
12405               as_Register($src1$$reg),
12406               as_Register($src2$$reg),
12407               Assembler::ASR,
12408               $src3$$constant & 0x1f);
12409   %}
12410 
12411   ins_pipe(ialu_reg_reg_shift);
12412 %}
12413 
12414 // This pattern is automatically generated from aarch64_ad.m4
12415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12416 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12417                          iRegL src1, iRegL src2,
12418                          immI src3) %{
12419   match(Set dst (XorL src1 (RShiftL src2 src3)));
12420 
12421   ins_cost(1.9 * INSN_COST);
12422   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12423 
12424   ins_encode %{
12425     __ eor(as_Register($dst$$reg),
12426               as_Register($src1$$reg),
12427               as_Register($src2$$reg),
12428               Assembler::ASR,
12429               $src3$$constant & 0x3f);
12430   %}
12431 
12432   ins_pipe(ialu_reg_reg_shift);
12433 %}
12434 
12435 // This pattern is automatically generated from aarch64_ad.m4
12436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12437 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12438                          iRegIorL2I src1, iRegIorL2I src2,
12439                          immI src3) %{
12440   match(Set dst (XorI src1 (LShiftI src2 src3)));
12441 
12442   ins_cost(1.9 * INSN_COST);
12443   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12444 
12445   ins_encode %{
12446     __ eorw(as_Register($dst$$reg),
12447               as_Register($src1$$reg),
12448               as_Register($src2$$reg),
12449               Assembler::LSL,
12450               $src3$$constant & 0x1f);
12451   %}
12452 
12453   ins_pipe(ialu_reg_reg_shift);
12454 %}
12455 
12456 // This pattern is automatically generated from aarch64_ad.m4
12457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12458 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12459                          iRegL src1, iRegL src2,
12460                          immI src3) %{
12461   match(Set dst (XorL src1 (LShiftL src2 src3)));
12462 
12463   ins_cost(1.9 * INSN_COST);
12464   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12465 
12466   ins_encode %{
12467     __ eor(as_Register($dst$$reg),
12468               as_Register($src1$$reg),
12469               as_Register($src2$$reg),
12470               Assembler::LSL,
12471               $src3$$constant & 0x3f);
12472   %}
12473 
12474   ins_pipe(ialu_reg_reg_shift);
12475 %}
12476 
12477 // This pattern is automatically generated from aarch64_ad.m4
12478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12479 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12480                          iRegIorL2I src1, iRegIorL2I src2,
12481                          immI src3) %{
12482   match(Set dst (XorI src1 (RotateRight src2 src3)));
12483 
12484   ins_cost(1.9 * INSN_COST);
12485   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12486 
12487   ins_encode %{
12488     __ eorw(as_Register($dst$$reg),
12489               as_Register($src1$$reg),
12490               as_Register($src2$$reg),
12491               Assembler::ROR,
12492               $src3$$constant & 0x1f);
12493   %}
12494 
12495   ins_pipe(ialu_reg_reg_shift);
12496 %}
12497 
12498 // This pattern is automatically generated from aarch64_ad.m4
12499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12500 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12501                          iRegL src1, iRegL src2,
12502                          immI src3) %{
12503   match(Set dst (XorL src1 (RotateRight src2 src3)));
12504 
12505   ins_cost(1.9 * INSN_COST);
12506   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12507 
12508   ins_encode %{
12509     __ eor(as_Register($dst$$reg),
12510               as_Register($src1$$reg),
12511               as_Register($src2$$reg),
12512               Assembler::ROR,
12513               $src3$$constant & 0x3f);
12514   %}
12515 
12516   ins_pipe(ialu_reg_reg_shift);
12517 %}
12518 
12519 // This pattern is automatically generated from aarch64_ad.m4
12520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12521 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12522                          iRegIorL2I src1, iRegIorL2I src2,
12523                          immI src3) %{
12524   match(Set dst (OrI src1 (URShiftI src2 src3)));
12525 
12526   ins_cost(1.9 * INSN_COST);
12527   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12528 
12529   ins_encode %{
12530     __ orrw(as_Register($dst$$reg),
12531               as_Register($src1$$reg),
12532               as_Register($src2$$reg),
12533               Assembler::LSR,
12534               $src3$$constant & 0x1f);
12535   %}
12536 
12537   ins_pipe(ialu_reg_reg_shift);
12538 %}
12539 
12540 // This pattern is automatically generated from aarch64_ad.m4
12541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12542 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12543                          iRegL src1, iRegL src2,
12544                          immI src3) %{
12545   match(Set dst (OrL src1 (URShiftL src2 src3)));
12546 
12547   ins_cost(1.9 * INSN_COST);
12548   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12549 
12550   ins_encode %{
12551     __ orr(as_Register($dst$$reg),
12552               as_Register($src1$$reg),
12553               as_Register($src2$$reg),
12554               Assembler::LSR,
12555               $src3$$constant & 0x3f);
12556   %}
12557 
12558   ins_pipe(ialu_reg_reg_shift);
12559 %}
12560 
12561 // This pattern is automatically generated from aarch64_ad.m4
12562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12563 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12564                          iRegIorL2I src1, iRegIorL2I src2,
12565                          immI src3) %{
12566   match(Set dst (OrI src1 (RShiftI src2 src3)));
12567 
12568   ins_cost(1.9 * INSN_COST);
12569   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12570 
12571   ins_encode %{
12572     __ orrw(as_Register($dst$$reg),
12573               as_Register($src1$$reg),
12574               as_Register($src2$$reg),
12575               Assembler::ASR,
12576               $src3$$constant & 0x1f);
12577   %}
12578 
12579   ins_pipe(ialu_reg_reg_shift);
12580 %}
12581 
12582 // This pattern is automatically generated from aarch64_ad.m4
12583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12584 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12585                          iRegL src1, iRegL src2,
12586                          immI src3) %{
12587   match(Set dst (OrL src1 (RShiftL src2 src3)));
12588 
12589   ins_cost(1.9 * INSN_COST);
12590   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12591 
12592   ins_encode %{
12593     __ orr(as_Register($dst$$reg),
12594               as_Register($src1$$reg),
12595               as_Register($src2$$reg),
12596               Assembler::ASR,
12597               $src3$$constant & 0x3f);
12598   %}
12599 
12600   ins_pipe(ialu_reg_reg_shift);
12601 %}
12602 
12603 // This pattern is automatically generated from aarch64_ad.m4
12604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12605 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12606                          iRegIorL2I src1, iRegIorL2I src2,
12607                          immI src3) %{
12608   match(Set dst (OrI src1 (LShiftI src2 src3)));
12609 
12610   ins_cost(1.9 * INSN_COST);
12611   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12612 
12613   ins_encode %{
12614     __ orrw(as_Register($dst$$reg),
12615               as_Register($src1$$reg),
12616               as_Register($src2$$reg),
12617               Assembler::LSL,
12618               $src3$$constant & 0x1f);
12619   %}
12620 
12621   ins_pipe(ialu_reg_reg_shift);
12622 %}
12623 
12624 // This pattern is automatically generated from aarch64_ad.m4
12625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12626 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12627                          iRegL src1, iRegL src2,
12628                          immI src3) %{
12629   match(Set dst (OrL src1 (LShiftL src2 src3)));
12630 
12631   ins_cost(1.9 * INSN_COST);
12632   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12633 
12634   ins_encode %{
12635     __ orr(as_Register($dst$$reg),
12636               as_Register($src1$$reg),
12637               as_Register($src2$$reg),
12638               Assembler::LSL,
12639               $src3$$constant & 0x3f);
12640   %}
12641 
12642   ins_pipe(ialu_reg_reg_shift);
12643 %}
12644 
12645 // This pattern is automatically generated from aarch64_ad.m4
12646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12647 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12648                          iRegIorL2I src1, iRegIorL2I src2,
12649                          immI src3) %{
12650   match(Set dst (OrI src1 (RotateRight src2 src3)));
12651 
12652   ins_cost(1.9 * INSN_COST);
12653   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12654 
12655   ins_encode %{
12656     __ orrw(as_Register($dst$$reg),
12657               as_Register($src1$$reg),
12658               as_Register($src2$$reg),
12659               Assembler::ROR,
12660               $src3$$constant & 0x1f);
12661   %}
12662 
12663   ins_pipe(ialu_reg_reg_shift);
12664 %}
12665 
12666 // This pattern is automatically generated from aarch64_ad.m4
12667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12668 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12669                          iRegL src1, iRegL src2,
12670                          immI src3) %{
12671   match(Set dst (OrL src1 (RotateRight src2 src3)));
12672 
12673   ins_cost(1.9 * INSN_COST);
12674   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12675 
12676   ins_encode %{
12677     __ orr(as_Register($dst$$reg),
12678               as_Register($src1$$reg),
12679               as_Register($src2$$reg),
12680               Assembler::ROR,
12681               $src3$$constant & 0x3f);
12682   %}
12683 
12684   ins_pipe(ialu_reg_reg_shift);
12685 %}
12686 
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12690                          iRegIorL2I src1, iRegIorL2I src2,
12691                          immI src3) %{
12692   match(Set dst (AddI src1 (URShiftI src2 src3)));
12693 
12694   ins_cost(1.9 * INSN_COST);
12695   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12696 
12697   ins_encode %{
12698     __ addw(as_Register($dst$$reg),
12699               as_Register($src1$$reg),
12700               as_Register($src2$$reg),
12701               Assembler::LSR,
12702               $src3$$constant & 0x1f);
12703   %}
12704 
12705   ins_pipe(ialu_reg_reg_shift);
12706 %}
12707 
12708 // This pattern is automatically generated from aarch64_ad.m4
12709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12710 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12711                          iRegL src1, iRegL src2,
12712                          immI src3) %{
12713   match(Set dst (AddL src1 (URShiftL src2 src3)));
12714 
12715   ins_cost(1.9 * INSN_COST);
12716   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12717 
12718   ins_encode %{
12719     __ add(as_Register($dst$$reg),
12720               as_Register($src1$$reg),
12721               as_Register($src2$$reg),
12722               Assembler::LSR,
12723               $src3$$constant & 0x3f);
12724   %}
12725 
12726   ins_pipe(ialu_reg_reg_shift);
12727 %}
12728 
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12732                          iRegIorL2I src1, iRegIorL2I src2,
12733                          immI src3) %{
12734   match(Set dst (AddI src1 (RShiftI src2 src3)));
12735 
12736   ins_cost(1.9 * INSN_COST);
12737   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12738 
12739   ins_encode %{
12740     __ addw(as_Register($dst$$reg),
12741               as_Register($src1$$reg),
12742               as_Register($src2$$reg),
12743               Assembler::ASR,
12744               $src3$$constant & 0x1f);
12745   %}
12746 
12747   ins_pipe(ialu_reg_reg_shift);
12748 %}
12749 
12750 // This pattern is automatically generated from aarch64_ad.m4
12751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12752 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12753                          iRegL src1, iRegL src2,
12754                          immI src3) %{
12755   match(Set dst (AddL src1 (RShiftL src2 src3)));
12756 
12757   ins_cost(1.9 * INSN_COST);
12758   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12759 
12760   ins_encode %{
12761     __ add(as_Register($dst$$reg),
12762               as_Register($src1$$reg),
12763               as_Register($src2$$reg),
12764               Assembler::ASR,
12765               $src3$$constant & 0x3f);
12766   %}
12767 
12768   ins_pipe(ialu_reg_reg_shift);
12769 %}
12770 
12771 // This pattern is automatically generated from aarch64_ad.m4
12772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12773 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12774                          iRegIorL2I src1, iRegIorL2I src2,
12775                          immI src3) %{
12776   match(Set dst (AddI src1 (LShiftI src2 src3)));
12777 
12778   ins_cost(1.9 * INSN_COST);
12779   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12780 
12781   ins_encode %{
12782     __ addw(as_Register($dst$$reg),
12783               as_Register($src1$$reg),
12784               as_Register($src2$$reg),
12785               Assembler::LSL,
12786               $src3$$constant & 0x1f);
12787   %}
12788 
12789   ins_pipe(ialu_reg_reg_shift);
12790 %}
12791 
12792 // This pattern is automatically generated from aarch64_ad.m4
12793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12794 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12795                          iRegL src1, iRegL src2,
12796                          immI src3) %{
12797   match(Set dst (AddL src1 (LShiftL src2 src3)));
12798 
12799   ins_cost(1.9 * INSN_COST);
12800   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12801 
12802   ins_encode %{
12803     __ add(as_Register($dst$$reg),
12804               as_Register($src1$$reg),
12805               as_Register($src2$$reg),
12806               Assembler::LSL,
12807               $src3$$constant & 0x3f);
12808   %}
12809 
12810   ins_pipe(ialu_reg_reg_shift);
12811 %}
12812 
12813 // This pattern is automatically generated from aarch64_ad.m4
12814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12815 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12816                          iRegIorL2I src1, iRegIorL2I src2,
12817                          immI src3) %{
12818   match(Set dst (SubI src1 (URShiftI src2 src3)));
12819 
12820   ins_cost(1.9 * INSN_COST);
12821   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12822 
12823   ins_encode %{
12824     __ subw(as_Register($dst$$reg),
12825               as_Register($src1$$reg),
12826               as_Register($src2$$reg),
12827               Assembler::LSR,
12828               $src3$$constant & 0x1f);
12829   %}
12830 
12831   ins_pipe(ialu_reg_reg_shift);
12832 %}
12833 
12834 // This pattern is automatically generated from aarch64_ad.m4
12835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12836 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12837                          iRegL src1, iRegL src2,
12838                          immI src3) %{
12839   match(Set dst (SubL src1 (URShiftL src2 src3)));
12840 
12841   ins_cost(1.9 * INSN_COST);
12842   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12843 
12844   ins_encode %{
12845     __ sub(as_Register($dst$$reg),
12846               as_Register($src1$$reg),
12847               as_Register($src2$$reg),
12848               Assembler::LSR,
12849               $src3$$constant & 0x3f);
12850   %}
12851 
12852   ins_pipe(ialu_reg_reg_shift);
12853 %}
12854 
12855 // This pattern is automatically generated from aarch64_ad.m4
12856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12857 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12858                          iRegIorL2I src1, iRegIorL2I src2,
12859                          immI src3) %{
12860   match(Set dst (SubI src1 (RShiftI src2 src3)));
12861 
12862   ins_cost(1.9 * INSN_COST);
12863   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12864 
12865   ins_encode %{
12866     __ subw(as_Register($dst$$reg),
12867               as_Register($src1$$reg),
12868               as_Register($src2$$reg),
12869               Assembler::ASR,
12870               $src3$$constant & 0x1f);
12871   %}
12872 
12873   ins_pipe(ialu_reg_reg_shift);
12874 %}
12875 
12876 // This pattern is automatically generated from aarch64_ad.m4
12877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12878 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12879                          iRegL src1, iRegL src2,
12880                          immI src3) %{
12881   match(Set dst (SubL src1 (RShiftL src2 src3)));
12882 
12883   ins_cost(1.9 * INSN_COST);
12884   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12885 
12886   ins_encode %{
12887     __ sub(as_Register($dst$$reg),
12888               as_Register($src1$$reg),
12889               as_Register($src2$$reg),
12890               Assembler::ASR,
12891               $src3$$constant & 0x3f);
12892   %}
12893 
12894   ins_pipe(ialu_reg_reg_shift);
12895 %}
12896 
12897 // This pattern is automatically generated from aarch64_ad.m4
12898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12899 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12900                          iRegIorL2I src1, iRegIorL2I src2,
12901                          immI src3) %{
12902   match(Set dst (SubI src1 (LShiftI src2 src3)));
12903 
12904   ins_cost(1.9 * INSN_COST);
12905   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12906 
12907   ins_encode %{
12908     __ subw(as_Register($dst$$reg),
12909               as_Register($src1$$reg),
12910               as_Register($src2$$reg),
12911               Assembler::LSL,
12912               $src3$$constant & 0x1f);
12913   %}
12914 
12915   ins_pipe(ialu_reg_reg_shift);
12916 %}
12917 
12918 // This pattern is automatically generated from aarch64_ad.m4
12919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12920 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12921                          iRegL src1, iRegL src2,
12922                          immI src3) %{
12923   match(Set dst (SubL src1 (LShiftL src2 src3)));
12924 
12925   ins_cost(1.9 * INSN_COST);
12926   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12927 
12928   ins_encode %{
12929     __ sub(as_Register($dst$$reg),
12930               as_Register($src1$$reg),
12931               as_Register($src2$$reg),
12932               Assembler::LSL,
12933               $src3$$constant & 0x3f);
12934   %}
12935 
12936   ins_pipe(ialu_reg_reg_shift);
12937 %}
12938 
12939 // This pattern is automatically generated from aarch64_ad.m4
12940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12941 
12942 // Shift Left followed by Shift Right.
12943 // This idiom is used by the compiler for the i2b bytecode etc.
12944 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12945 %{
12946   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12947   ins_cost(INSN_COST * 2);
12948   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12949   ins_encode %{
12950     int lshift = $lshift_count$$constant & 63;
12951     int rshift = $rshift_count$$constant & 63;
12952     int s = 63 - lshift;
12953     int r = (rshift - lshift) & 63;
12954     __ sbfm(as_Register($dst$$reg),
12955             as_Register($src$$reg),
12956             r, s);
12957   %}
12958 
12959   ins_pipe(ialu_reg_shift);
12960 %}
12961 
12962 // This pattern is automatically generated from aarch64_ad.m4
12963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12964 
12965 // Shift Left followed by Shift Right.
12966 // This idiom is used by the compiler for the i2b bytecode etc.
12967 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12968 %{
12969   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12970   ins_cost(INSN_COST * 2);
12971   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12972   ins_encode %{
12973     int lshift = $lshift_count$$constant & 31;
12974     int rshift = $rshift_count$$constant & 31;
12975     int s = 31 - lshift;
12976     int r = (rshift - lshift) & 31;
12977     __ sbfmw(as_Register($dst$$reg),
12978             as_Register($src$$reg),
12979             r, s);
12980   %}
12981 
12982   ins_pipe(ialu_reg_shift);
12983 %}
12984 
12985 // This pattern is automatically generated from aarch64_ad.m4
12986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12987 
12988 // Shift Left followed by Shift Right.
12989 // This idiom is used by the compiler for the i2b bytecode etc.
12990 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12991 %{
12992   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12993   ins_cost(INSN_COST * 2);
12994   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12995   ins_encode %{
12996     int lshift = $lshift_count$$constant & 63;
12997     int rshift = $rshift_count$$constant & 63;
12998     int s = 63 - lshift;
12999     int r = (rshift - lshift) & 63;
13000     __ ubfm(as_Register($dst$$reg),
13001             as_Register($src$$reg),
13002             r, s);
13003   %}
13004 
13005   ins_pipe(ialu_reg_shift);
13006 %}
13007 
13008 // This pattern is automatically generated from aarch64_ad.m4
13009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13010 
13011 // Shift Left followed by Shift Right.
13012 // This idiom is used by the compiler for the i2b bytecode etc.
13013 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13014 %{
13015   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13016   ins_cost(INSN_COST * 2);
13017   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13018   ins_encode %{
13019     int lshift = $lshift_count$$constant & 31;
13020     int rshift = $rshift_count$$constant & 31;
13021     int s = 31 - lshift;
13022     int r = (rshift - lshift) & 31;
13023     __ ubfmw(as_Register($dst$$reg),
13024             as_Register($src$$reg),
13025             r, s);
13026   %}
13027 
13028   ins_pipe(ialu_reg_shift);
13029 %}
13030 
13031 // Bitfield extract with shift & mask
13032 
13033 // This pattern is automatically generated from aarch64_ad.m4
13034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13035 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13036 %{
13037   match(Set dst (AndI (URShiftI src rshift) mask));
13038   // Make sure we are not going to exceed what ubfxw can do.
13039   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13040 
13041   ins_cost(INSN_COST);
13042   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13043   ins_encode %{
13044     int rshift = $rshift$$constant & 31;
13045     intptr_t mask = $mask$$constant;
13046     int width = exact_log2(mask+1);
13047     __ ubfxw(as_Register($dst$$reg),
13048             as_Register($src$$reg), rshift, width);
13049   %}
13050   ins_pipe(ialu_reg_shift);
13051 %}
13052 
13053 // This pattern is automatically generated from aarch64_ad.m4
13054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13055 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13056 %{
13057   match(Set dst (AndL (URShiftL src rshift) mask));
13058   // Make sure we are not going to exceed what ubfx can do.
13059   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13060 
13061   ins_cost(INSN_COST);
13062   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13063   ins_encode %{
13064     int rshift = $rshift$$constant & 63;
13065     intptr_t mask = $mask$$constant;
13066     int width = exact_log2_long(mask+1);
13067     __ ubfx(as_Register($dst$$reg),
13068             as_Register($src$$reg), rshift, width);
13069   %}
13070   ins_pipe(ialu_reg_shift);
13071 %}
13072 
13073 
13074 // This pattern is automatically generated from aarch64_ad.m4
13075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13076 
13077 // We can use ubfx when extending an And with a mask when we know mask
13078 // is positive.  We know that because immI_bitmask guarantees it.
13079 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13080 %{
13081   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13082   // Make sure we are not going to exceed what ubfxw can do.
13083   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13084 
13085   ins_cost(INSN_COST * 2);
13086   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13087   ins_encode %{
13088     int rshift = $rshift$$constant & 31;
13089     intptr_t mask = $mask$$constant;
13090     int width = exact_log2(mask+1);
13091     __ ubfx(as_Register($dst$$reg),
13092             as_Register($src$$reg), rshift, width);
13093   %}
13094   ins_pipe(ialu_reg_shift);
13095 %}
13096 
13097 
13098 // This pattern is automatically generated from aarch64_ad.m4
13099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13100 
13101 // We can use ubfiz when masking by a positive number and then left shifting the result.
13102 // We know that the mask is positive because immI_bitmask guarantees it.
13103 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13104 %{
13105   match(Set dst (LShiftI (AndI src mask) lshift));
13106   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13107 
13108   ins_cost(INSN_COST);
13109   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13110   ins_encode %{
13111     int lshift = $lshift$$constant & 31;
13112     intptr_t mask = $mask$$constant;
13113     int width = exact_log2(mask+1);
13114     __ ubfizw(as_Register($dst$$reg),
13115           as_Register($src$$reg), lshift, width);
13116   %}
13117   ins_pipe(ialu_reg_shift);
13118 %}
13119 
13120 // This pattern is automatically generated from aarch64_ad.m4
13121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13122 
13123 // We can use ubfiz when masking by a positive number and then left shifting the result.
13124 // We know that the mask is positive because immL_bitmask guarantees it.
13125 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13126 %{
13127   match(Set dst (LShiftL (AndL src mask) lshift));
13128   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13129 
13130   ins_cost(INSN_COST);
13131   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13132   ins_encode %{
13133     int lshift = $lshift$$constant & 63;
13134     intptr_t mask = $mask$$constant;
13135     int width = exact_log2_long(mask+1);
13136     __ ubfiz(as_Register($dst$$reg),
13137           as_Register($src$$reg), lshift, width);
13138   %}
13139   ins_pipe(ialu_reg_shift);
13140 %}
13141 
13142 // This pattern is automatically generated from aarch64_ad.m4
13143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13144 
13145 // We can use ubfiz when masking by a positive number and then left shifting the result.
13146 // We know that the mask is positive because immI_bitmask guarantees it.
13147 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13148 %{
13149   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13150   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13151 
13152   ins_cost(INSN_COST);
13153   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13154   ins_encode %{
13155     int lshift = $lshift$$constant & 31;
13156     intptr_t mask = $mask$$constant;
13157     int width = exact_log2(mask+1);
13158     __ ubfizw(as_Register($dst$$reg),
13159           as_Register($src$$reg), lshift, width);
13160   %}
13161   ins_pipe(ialu_reg_shift);
13162 %}
13163 
13164 // This pattern is automatically generated from aarch64_ad.m4
13165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13166 
13167 // We can use ubfiz when masking by a positive number and then left shifting the result.
13168 // We know that the mask is positive because immL_bitmask guarantees it.
13169 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13170 %{
13171   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13172   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13173 
13174   ins_cost(INSN_COST);
13175   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13176   ins_encode %{
13177     int lshift = $lshift$$constant & 63;
13178     intptr_t mask = $mask$$constant;
13179     int width = exact_log2_long(mask+1);
13180     __ ubfiz(as_Register($dst$$reg),
13181           as_Register($src$$reg), lshift, width);
13182   %}
13183   ins_pipe(ialu_reg_shift);
13184 %}
13185 
13186 
13187 // This pattern is automatically generated from aarch64_ad.m4
13188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13189 
13190 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13191 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13192 %{
13193   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13194   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13195 
13196   ins_cost(INSN_COST);
13197   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13198   ins_encode %{
13199     int lshift = $lshift$$constant & 63;
13200     intptr_t mask = $mask$$constant;
13201     int width = exact_log2(mask+1);
13202     __ ubfiz(as_Register($dst$$reg),
13203              as_Register($src$$reg), lshift, width);
13204   %}
13205   ins_pipe(ialu_reg_shift);
13206 %}
13207 
13208 // This pattern is automatically generated from aarch64_ad.m4
13209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13210 
13211 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13212 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13213 %{
13214   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13215   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13216 
13217   ins_cost(INSN_COST);
13218   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13219   ins_encode %{
13220     int lshift = $lshift$$constant & 31;
13221     intptr_t mask = $mask$$constant;
13222     int width = exact_log2(mask+1);
13223     __ ubfiz(as_Register($dst$$reg),
13224              as_Register($src$$reg), lshift, width);
13225   %}
13226   ins_pipe(ialu_reg_shift);
13227 %}
13228 
13229 // This pattern is automatically generated from aarch64_ad.m4
13230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13231 
13232 // Can skip int2long conversions after AND with small bitmask
13233 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13234 %{
13235   match(Set dst (ConvI2L (AndI src msk)));
13236   ins_cost(INSN_COST);
13237   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13238   ins_encode %{
13239     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13240   %}
13241   ins_pipe(ialu_reg_shift);
13242 %}
13243 
13244 
13245 // Rotations
13246 
13247 // This pattern is automatically generated from aarch64_ad.m4
13248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13249 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13250 %{
13251   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13252   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13253 
13254   ins_cost(INSN_COST);
13255   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13256 
13257   ins_encode %{
13258     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13259             $rshift$$constant & 63);
13260   %}
13261   ins_pipe(ialu_reg_reg_extr);
13262 %}
13263 
13264 
13265 // This pattern is automatically generated from aarch64_ad.m4
13266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13267 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13268 %{
13269   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13270   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13271 
13272   ins_cost(INSN_COST);
13273   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13274 
13275   ins_encode %{
13276     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13277             $rshift$$constant & 31);
13278   %}
13279   ins_pipe(ialu_reg_reg_extr);
13280 %}
13281 
13282 
13283 // This pattern is automatically generated from aarch64_ad.m4
13284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13285 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13286 %{
13287   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13288   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13289 
13290   ins_cost(INSN_COST);
13291   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13292 
13293   ins_encode %{
13294     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13295             $rshift$$constant & 63);
13296   %}
13297   ins_pipe(ialu_reg_reg_extr);
13298 %}
13299 
13300 
13301 // This pattern is automatically generated from aarch64_ad.m4
13302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13303 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13304 %{
13305   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13306   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13307 
13308   ins_cost(INSN_COST);
13309   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13310 
13311   ins_encode %{
13312     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13313             $rshift$$constant & 31);
13314   %}
13315   ins_pipe(ialu_reg_reg_extr);
13316 %}
13317 
13318 // This pattern is automatically generated from aarch64_ad.m4
13319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13320 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13321 %{
13322   match(Set dst (RotateRight src shift));
13323 
13324   ins_cost(INSN_COST);
13325   format %{ "ror    $dst, $src, $shift" %}
13326 
13327   ins_encode %{
13328      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13329                $shift$$constant & 0x1f);
13330   %}
13331   ins_pipe(ialu_reg_reg_vshift);
13332 %}
13333 
13334 // This pattern is automatically generated from aarch64_ad.m4
13335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13336 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13337 %{
13338   match(Set dst (RotateRight src shift));
13339 
13340   ins_cost(INSN_COST);
13341   format %{ "ror    $dst, $src, $shift" %}
13342 
13343   ins_encode %{
13344      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13345                $shift$$constant & 0x3f);
13346   %}
13347   ins_pipe(ialu_reg_reg_vshift);
13348 %}
13349 
13350 // This pattern is automatically generated from aarch64_ad.m4
13351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13352 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13353 %{
13354   match(Set dst (RotateRight src shift));
13355 
13356   ins_cost(INSN_COST);
13357   format %{ "ror    $dst, $src, $shift" %}
13358 
13359   ins_encode %{
13360      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13361   %}
13362   ins_pipe(ialu_reg_reg_vshift);
13363 %}
13364 
13365 // This pattern is automatically generated from aarch64_ad.m4
13366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13367 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13368 %{
13369   match(Set dst (RotateRight src shift));
13370 
13371   ins_cost(INSN_COST);
13372   format %{ "ror    $dst, $src, $shift" %}
13373 
13374   ins_encode %{
13375      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13376   %}
13377   ins_pipe(ialu_reg_reg_vshift);
13378 %}
13379 
13380 // This pattern is automatically generated from aarch64_ad.m4
13381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13382 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13383 %{
13384   match(Set dst (RotateLeft src shift));
13385 
13386   ins_cost(INSN_COST);
13387   format %{ "rol    $dst, $src, $shift" %}
13388 
13389   ins_encode %{
13390      __ subw(rscratch1, zr, as_Register($shift$$reg));
13391      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13392   %}
13393   ins_pipe(ialu_reg_reg_vshift);
13394 %}
13395 
13396 // This pattern is automatically generated from aarch64_ad.m4
13397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13398 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13399 %{
13400   match(Set dst (RotateLeft src shift));
13401 
13402   ins_cost(INSN_COST);
13403   format %{ "rol    $dst, $src, $shift" %}
13404 
13405   ins_encode %{
13406      __ subw(rscratch1, zr, as_Register($shift$$reg));
13407      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13408   %}
13409   ins_pipe(ialu_reg_reg_vshift);
13410 %}
13411 
13412 
13413 // Add/subtract (extended)
13414 
13415 // This pattern is automatically generated from aarch64_ad.m4
13416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13417 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13418 %{
13419   match(Set dst (AddL src1 (ConvI2L src2)));
13420   ins_cost(INSN_COST);
13421   format %{ "add  $dst, $src1, $src2, sxtw" %}
13422 
13423    ins_encode %{
13424      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13425             as_Register($src2$$reg), ext::sxtw);
13426    %}
13427   ins_pipe(ialu_reg_reg);
13428 %}
13429 
13430 // This pattern is automatically generated from aarch64_ad.m4
13431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13432 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13433 %{
13434   match(Set dst (SubL src1 (ConvI2L src2)));
13435   ins_cost(INSN_COST);
13436   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13437 
13438    ins_encode %{
13439      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13440             as_Register($src2$$reg), ext::sxtw);
13441    %}
13442   ins_pipe(ialu_reg_reg);
13443 %}
13444 
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13448 %{
13449   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13450   ins_cost(INSN_COST);
13451   format %{ "add  $dst, $src1, $src2, sxth" %}
13452 
13453    ins_encode %{
13454      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13455             as_Register($src2$$reg), ext::sxth);
13456    %}
13457   ins_pipe(ialu_reg_reg);
13458 %}
13459 
13460 // This pattern is automatically generated from aarch64_ad.m4
13461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13462 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13463 %{
13464   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13465   ins_cost(INSN_COST);
13466   format %{ "add  $dst, $src1, $src2, sxtb" %}
13467 
13468    ins_encode %{
13469      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13470             as_Register($src2$$reg), ext::sxtb);
13471    %}
13472   ins_pipe(ialu_reg_reg);
13473 %}
13474 
13475 // This pattern is automatically generated from aarch64_ad.m4
13476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13477 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13478 %{
13479   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13480   ins_cost(INSN_COST);
13481   format %{ "add  $dst, $src1, $src2, uxtb" %}
13482 
13483    ins_encode %{
13484      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13485             as_Register($src2$$reg), ext::uxtb);
13486    %}
13487   ins_pipe(ialu_reg_reg);
13488 %}
13489 
13490 // This pattern is automatically generated from aarch64_ad.m4
13491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13492 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13493 %{
13494   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13495   ins_cost(INSN_COST);
13496   format %{ "add  $dst, $src1, $src2, sxth" %}
13497 
13498    ins_encode %{
13499      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13500             as_Register($src2$$reg), ext::sxth);
13501    %}
13502   ins_pipe(ialu_reg_reg);
13503 %}
13504 
13505 // This pattern is automatically generated from aarch64_ad.m4
13506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13507 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13508 %{
13509   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13510   ins_cost(INSN_COST);
13511   format %{ "add  $dst, $src1, $src2, sxtw" %}
13512 
13513    ins_encode %{
13514      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13515             as_Register($src2$$reg), ext::sxtw);
13516    %}
13517   ins_pipe(ialu_reg_reg);
13518 %}
13519 
13520 // This pattern is automatically generated from aarch64_ad.m4
13521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13522 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13523 %{
13524   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13525   ins_cost(INSN_COST);
13526   format %{ "add  $dst, $src1, $src2, sxtb" %}
13527 
13528    ins_encode %{
13529      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13530             as_Register($src2$$reg), ext::sxtb);
13531    %}
13532   ins_pipe(ialu_reg_reg);
13533 %}
13534 
13535 // This pattern is automatically generated from aarch64_ad.m4
13536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13537 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13538 %{
13539   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13540   ins_cost(INSN_COST);
13541   format %{ "add  $dst, $src1, $src2, uxtb" %}
13542 
13543    ins_encode %{
13544      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13545             as_Register($src2$$reg), ext::uxtb);
13546    %}
13547   ins_pipe(ialu_reg_reg);
13548 %}
13549 
13550 // This pattern is automatically generated from aarch64_ad.m4
13551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13552 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13553 %{
13554   match(Set dst (AddI src1 (AndI src2 mask)));
13555   ins_cost(INSN_COST);
13556   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13557 
13558    ins_encode %{
13559      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13560             as_Register($src2$$reg), ext::uxtb);
13561    %}
13562   ins_pipe(ialu_reg_reg);
13563 %}
13564 
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13568 %{
13569   match(Set dst (AddI src1 (AndI src2 mask)));
13570   ins_cost(INSN_COST);
13571   format %{ "addw  $dst, $src1, $src2, uxth" %}
13572 
13573    ins_encode %{
13574      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13575             as_Register($src2$$reg), ext::uxth);
13576    %}
13577   ins_pipe(ialu_reg_reg);
13578 %}
13579 
13580 // This pattern is automatically generated from aarch64_ad.m4
13581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13582 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13583 %{
13584   match(Set dst (AddL src1 (AndL src2 mask)));
13585   ins_cost(INSN_COST);
13586   format %{ "add  $dst, $src1, $src2, uxtb" %}
13587 
13588    ins_encode %{
13589      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13590             as_Register($src2$$reg), ext::uxtb);
13591    %}
13592   ins_pipe(ialu_reg_reg);
13593 %}
13594 
13595 // This pattern is automatically generated from aarch64_ad.m4
13596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13597 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13598 %{
13599   match(Set dst (AddL src1 (AndL src2 mask)));
13600   ins_cost(INSN_COST);
13601   format %{ "add  $dst, $src1, $src2, uxth" %}
13602 
13603    ins_encode %{
13604      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13605             as_Register($src2$$reg), ext::uxth);
13606    %}
13607   ins_pipe(ialu_reg_reg);
13608 %}
13609 
13610 // This pattern is automatically generated from aarch64_ad.m4
13611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13612 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13613 %{
13614   match(Set dst (AddL src1 (AndL src2 mask)));
13615   ins_cost(INSN_COST);
13616   format %{ "add  $dst, $src1, $src2, uxtw" %}
13617 
13618    ins_encode %{
13619      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13620             as_Register($src2$$reg), ext::uxtw);
13621    %}
13622   ins_pipe(ialu_reg_reg);
13623 %}
13624 
13625 // This pattern is automatically generated from aarch64_ad.m4
13626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13627 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13628 %{
13629   match(Set dst (SubI src1 (AndI src2 mask)));
13630   ins_cost(INSN_COST);
13631   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13632 
13633    ins_encode %{
13634      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13635             as_Register($src2$$reg), ext::uxtb);
13636    %}
13637   ins_pipe(ialu_reg_reg);
13638 %}
13639 
13640 // This pattern is automatically generated from aarch64_ad.m4
13641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13642 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13643 %{
13644   match(Set dst (SubI src1 (AndI src2 mask)));
13645   ins_cost(INSN_COST);
13646   format %{ "subw  $dst, $src1, $src2, uxth" %}
13647 
13648    ins_encode %{
13649      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13650             as_Register($src2$$reg), ext::uxth);
13651    %}
13652   ins_pipe(ialu_reg_reg);
13653 %}
13654 
13655 // This pattern is automatically generated from aarch64_ad.m4
13656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13657 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13658 %{
13659   match(Set dst (SubL src1 (AndL src2 mask)));
13660   ins_cost(INSN_COST);
13661   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13662 
13663    ins_encode %{
13664      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13665             as_Register($src2$$reg), ext::uxtb);
13666    %}
13667   ins_pipe(ialu_reg_reg);
13668 %}
13669 
13670 // This pattern is automatically generated from aarch64_ad.m4
13671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13672 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13673 %{
13674   match(Set dst (SubL src1 (AndL src2 mask)));
13675   ins_cost(INSN_COST);
13676   format %{ "sub  $dst, $src1, $src2, uxth" %}
13677 
13678    ins_encode %{
13679      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13680             as_Register($src2$$reg), ext::uxth);
13681    %}
13682   ins_pipe(ialu_reg_reg);
13683 %}
13684 
13685 // This pattern is automatically generated from aarch64_ad.m4
13686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13687 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13688 %{
13689   match(Set dst (SubL src1 (AndL src2 mask)));
13690   ins_cost(INSN_COST);
13691   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13692 
13693    ins_encode %{
13694      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13695             as_Register($src2$$reg), ext::uxtw);
13696    %}
13697   ins_pipe(ialu_reg_reg);
13698 %}
13699 
13700 
13701 // This pattern is automatically generated from aarch64_ad.m4
13702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13703 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13704 %{
13705   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13706   ins_cost(1.9 * INSN_COST);
13707   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13708 
13709    ins_encode %{
13710      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13711             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13712    %}
13713   ins_pipe(ialu_reg_reg_shift);
13714 %}
13715 
13716 // This pattern is automatically generated from aarch64_ad.m4
13717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13718 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13719 %{
13720   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13721   ins_cost(1.9 * INSN_COST);
13722   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13723 
13724    ins_encode %{
13725      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13726             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13727    %}
13728   ins_pipe(ialu_reg_reg_shift);
13729 %}
13730 
13731 // This pattern is automatically generated from aarch64_ad.m4
13732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13733 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13734 %{
13735   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13736   ins_cost(1.9 * INSN_COST);
13737   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13738 
13739    ins_encode %{
13740      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13741             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13742    %}
13743   ins_pipe(ialu_reg_reg_shift);
13744 %}
13745 
13746 // This pattern is automatically generated from aarch64_ad.m4
13747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13748 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13749 %{
13750   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13751   ins_cost(1.9 * INSN_COST);
13752   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13753 
13754    ins_encode %{
13755      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13756             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13757    %}
13758   ins_pipe(ialu_reg_reg_shift);
13759 %}
13760 
13761 // This pattern is automatically generated from aarch64_ad.m4
13762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13763 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13764 %{
13765   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13766   ins_cost(1.9 * INSN_COST);
13767   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13768 
13769    ins_encode %{
13770      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13771             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13772    %}
13773   ins_pipe(ialu_reg_reg_shift);
13774 %}
13775 
13776 // This pattern is automatically generated from aarch64_ad.m4
13777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13778 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13779 %{
13780   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13781   ins_cost(1.9 * INSN_COST);
13782   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13783 
13784    ins_encode %{
13785      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13786             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13787    %}
13788   ins_pipe(ialu_reg_reg_shift);
13789 %}
13790 
13791 // This pattern is automatically generated from aarch64_ad.m4
13792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13793 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13794 %{
13795   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13796   ins_cost(1.9 * INSN_COST);
13797   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13798 
13799    ins_encode %{
13800      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13801             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13802    %}
13803   ins_pipe(ialu_reg_reg_shift);
13804 %}
13805 
13806 // This pattern is automatically generated from aarch64_ad.m4
13807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13808 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13809 %{
13810   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13811   ins_cost(1.9 * INSN_COST);
13812   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13813 
13814    ins_encode %{
13815      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13816             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13817    %}
13818   ins_pipe(ialu_reg_reg_shift);
13819 %}
13820 
13821 // This pattern is automatically generated from aarch64_ad.m4
13822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13823 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13824 %{
13825   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13826   ins_cost(1.9 * INSN_COST);
13827   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13828 
13829    ins_encode %{
13830      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13831             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13832    %}
13833   ins_pipe(ialu_reg_reg_shift);
13834 %}
13835 
13836 // This pattern is automatically generated from aarch64_ad.m4
13837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13838 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13839 %{
13840   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13841   ins_cost(1.9 * INSN_COST);
13842   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13843 
13844    ins_encode %{
13845      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13846             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13847    %}
13848   ins_pipe(ialu_reg_reg_shift);
13849 %}
13850 
13851 // This pattern is automatically generated from aarch64_ad.m4
13852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13853 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13854 %{
13855   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13856   ins_cost(1.9 * INSN_COST);
13857   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13858 
13859    ins_encode %{
13860      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13861             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13862    %}
13863   ins_pipe(ialu_reg_reg_shift);
13864 %}
13865 
13866 // This pattern is automatically generated from aarch64_ad.m4
13867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13868 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13869 %{
13870   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13871   ins_cost(1.9 * INSN_COST);
13872   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13873 
13874    ins_encode %{
13875      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13876             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13877    %}
13878   ins_pipe(ialu_reg_reg_shift);
13879 %}
13880 
13881 // This pattern is automatically generated from aarch64_ad.m4
13882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13883 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13884 %{
13885   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13886   ins_cost(1.9 * INSN_COST);
13887   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13888 
13889    ins_encode %{
13890      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13891             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13892    %}
13893   ins_pipe(ialu_reg_reg_shift);
13894 %}
13895 
13896 // This pattern is automatically generated from aarch64_ad.m4
13897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13898 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13899 %{
13900   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13901   ins_cost(1.9 * INSN_COST);
13902   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13903 
13904    ins_encode %{
13905      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13906             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13907    %}
13908   ins_pipe(ialu_reg_reg_shift);
13909 %}
13910 
13911 // This pattern is automatically generated from aarch64_ad.m4
13912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13913 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13914 %{
13915   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13916   ins_cost(1.9 * INSN_COST);
13917   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13918 
13919    ins_encode %{
13920      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13921             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13922    %}
13923   ins_pipe(ialu_reg_reg_shift);
13924 %}
13925 
13926 // This pattern is automatically generated from aarch64_ad.m4
13927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13928 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13929 %{
13930   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13931   ins_cost(1.9 * INSN_COST);
13932   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13933 
13934    ins_encode %{
13935      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13936             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13937    %}
13938   ins_pipe(ialu_reg_reg_shift);
13939 %}
13940 
13941 // This pattern is automatically generated from aarch64_ad.m4
13942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13943 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13944 %{
13945   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13946   ins_cost(1.9 * INSN_COST);
13947   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13948 
13949    ins_encode %{
13950      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13951             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13952    %}
13953   ins_pipe(ialu_reg_reg_shift);
13954 %}
13955 
13956 // This pattern is automatically generated from aarch64_ad.m4
13957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13958 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13959 %{
13960   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13961   ins_cost(1.9 * INSN_COST);
13962   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13963 
13964    ins_encode %{
13965      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13966             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13967    %}
13968   ins_pipe(ialu_reg_reg_shift);
13969 %}
13970 
13971 // This pattern is automatically generated from aarch64_ad.m4
13972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13973 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13974 %{
13975   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13976   ins_cost(1.9 * INSN_COST);
13977   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13978 
13979    ins_encode %{
13980      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13981             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13982    %}
13983   ins_pipe(ialu_reg_reg_shift);
13984 %}
13985 
13986 // This pattern is automatically generated from aarch64_ad.m4
13987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13988 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13989 %{
13990   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13991   ins_cost(1.9 * INSN_COST);
13992   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13993 
13994    ins_encode %{
13995      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13996             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13997    %}
13998   ins_pipe(ialu_reg_reg_shift);
13999 %}
14000 
14001 // This pattern is automatically generated from aarch64_ad.m4
14002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14003 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14004 %{
14005   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14006   ins_cost(1.9 * INSN_COST);
14007   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
14008 
14009    ins_encode %{
14010      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14011             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14012    %}
14013   ins_pipe(ialu_reg_reg_shift);
14014 %}
14015 
14016 // This pattern is automatically generated from aarch64_ad.m4
14017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14018 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14019 %{
14020   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14021   ins_cost(1.9 * INSN_COST);
14022   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
14023 
14024    ins_encode %{
14025      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14026             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14027    %}
14028   ins_pipe(ialu_reg_reg_shift);
14029 %}
14030 
14031 // This pattern is automatically generated from aarch64_ad.m4
14032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14033 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14034 %{
14035   effect(DEF dst, USE src1, USE src2, USE cr);
14036   ins_cost(INSN_COST * 2);
14037   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
14038 
14039   ins_encode %{
14040     __ cselw($dst$$Register,
14041              $src1$$Register,
14042              $src2$$Register,
14043              Assembler::LT);
14044   %}
14045   ins_pipe(icond_reg_reg);
14046 %}
14047 
14048 // This pattern is automatically generated from aarch64_ad.m4
14049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14050 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14051 %{
14052   effect(DEF dst, USE src1, USE src2, USE cr);
14053   ins_cost(INSN_COST * 2);
14054   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
14055 
14056   ins_encode %{
14057     __ cselw($dst$$Register,
14058              $src1$$Register,
14059              $src2$$Register,
14060              Assembler::GT);
14061   %}
14062   ins_pipe(icond_reg_reg);
14063 %}
14064 
14065 // This pattern is automatically generated from aarch64_ad.m4
14066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14067 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14068 %{
14069   effect(DEF dst, USE src1, USE cr);
14070   ins_cost(INSN_COST * 2);
14071   format %{ "cselw $dst, $src1, zr lt\t"  %}
14072 
14073   ins_encode %{
14074     __ cselw($dst$$Register,
14075              $src1$$Register,
14076              zr,
14077              Assembler::LT);
14078   %}
14079   ins_pipe(icond_reg);
14080 %}
14081 
14082 // This pattern is automatically generated from aarch64_ad.m4
14083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14084 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14085 %{
14086   effect(DEF dst, USE src1, USE cr);
14087   ins_cost(INSN_COST * 2);
14088   format %{ "cselw $dst, $src1, zr gt\t"  %}
14089 
14090   ins_encode %{
14091     __ cselw($dst$$Register,
14092              $src1$$Register,
14093              zr,
14094              Assembler::GT);
14095   %}
14096   ins_pipe(icond_reg);
14097 %}
14098 
14099 // This pattern is automatically generated from aarch64_ad.m4
14100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14101 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14102 %{
14103   effect(DEF dst, USE src1, USE cr);
14104   ins_cost(INSN_COST * 2);
14105   format %{ "csincw $dst, $src1, zr le\t"  %}
14106 
14107   ins_encode %{
14108     __ csincw($dst$$Register,
14109              $src1$$Register,
14110              zr,
14111              Assembler::LE);
14112   %}
14113   ins_pipe(icond_reg);
14114 %}
14115 
14116 // This pattern is automatically generated from aarch64_ad.m4
14117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14118 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14119 %{
14120   effect(DEF dst, USE src1, USE cr);
14121   ins_cost(INSN_COST * 2);
14122   format %{ "csincw $dst, $src1, zr gt\t"  %}
14123 
14124   ins_encode %{
14125     __ csincw($dst$$Register,
14126              $src1$$Register,
14127              zr,
14128              Assembler::GT);
14129   %}
14130   ins_pipe(icond_reg);
14131 %}
14132 
14133 // This pattern is automatically generated from aarch64_ad.m4
14134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14135 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14136 %{
14137   effect(DEF dst, USE src1, USE cr);
14138   ins_cost(INSN_COST * 2);
14139   format %{ "csinvw $dst, $src1, zr lt\t"  %}
14140 
14141   ins_encode %{
14142     __ csinvw($dst$$Register,
14143              $src1$$Register,
14144              zr,
14145              Assembler::LT);
14146   %}
14147   ins_pipe(icond_reg);
14148 %}
14149 
14150 // This pattern is automatically generated from aarch64_ad.m4
14151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14152 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14153 %{
14154   effect(DEF dst, USE src1, USE cr);
14155   ins_cost(INSN_COST * 2);
14156   format %{ "csinvw $dst, $src1, zr ge\t"  %}
14157 
14158   ins_encode %{
14159     __ csinvw($dst$$Register,
14160              $src1$$Register,
14161              zr,
14162              Assembler::GE);
14163   %}
14164   ins_pipe(icond_reg);
14165 %}
14166 
14167 // This pattern is automatically generated from aarch64_ad.m4
14168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14169 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14170 %{
14171   match(Set dst (MinI src imm));
14172   ins_cost(INSN_COST * 3);
14173   expand %{
14174     rFlagsReg cr;
14175     compI_reg_imm0(cr, src);
14176     cmovI_reg_imm0_lt(dst, src, cr);
14177   %}
14178 %}
14179 
14180 // This pattern is automatically generated from aarch64_ad.m4
14181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14182 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14183 %{
14184   match(Set dst (MinI imm src));
14185   ins_cost(INSN_COST * 3);
14186   expand %{
14187     rFlagsReg cr;
14188     compI_reg_imm0(cr, src);
14189     cmovI_reg_imm0_lt(dst, src, cr);
14190   %}
14191 %}
14192 
14193 // This pattern is automatically generated from aarch64_ad.m4
14194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14195 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14196 %{
14197   match(Set dst (MinI src imm));
14198   ins_cost(INSN_COST * 3);
14199   expand %{
14200     rFlagsReg cr;
14201     compI_reg_imm0(cr, src);
14202     cmovI_reg_imm1_le(dst, src, cr);
14203   %}
14204 %}
14205 
14206 // This pattern is automatically generated from aarch64_ad.m4
14207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14208 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14209 %{
14210   match(Set dst (MinI imm src));
14211   ins_cost(INSN_COST * 3);
14212   expand %{
14213     rFlagsReg cr;
14214     compI_reg_imm0(cr, src);
14215     cmovI_reg_imm1_le(dst, src, cr);
14216   %}
14217 %}
14218 
14219 // This pattern is automatically generated from aarch64_ad.m4
14220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14221 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14222 %{
14223   match(Set dst (MinI src imm));
14224   ins_cost(INSN_COST * 3);
14225   expand %{
14226     rFlagsReg cr;
14227     compI_reg_imm0(cr, src);
14228     cmovI_reg_immM1_lt(dst, src, cr);
14229   %}
14230 %}
14231 
14232 // This pattern is automatically generated from aarch64_ad.m4
14233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14234 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14235 %{
14236   match(Set dst (MinI imm src));
14237   ins_cost(INSN_COST * 3);
14238   expand %{
14239     rFlagsReg cr;
14240     compI_reg_imm0(cr, src);
14241     cmovI_reg_immM1_lt(dst, src, cr);
14242   %}
14243 %}
14244 
14245 // This pattern is automatically generated from aarch64_ad.m4
14246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14247 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14248 %{
14249   match(Set dst (MaxI src imm));
14250   ins_cost(INSN_COST * 3);
14251   expand %{
14252     rFlagsReg cr;
14253     compI_reg_imm0(cr, src);
14254     cmovI_reg_imm0_gt(dst, src, cr);
14255   %}
14256 %}
14257 
14258 // This pattern is automatically generated from aarch64_ad.m4
14259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14260 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14261 %{
14262   match(Set dst (MaxI imm src));
14263   ins_cost(INSN_COST * 3);
14264   expand %{
14265     rFlagsReg cr;
14266     compI_reg_imm0(cr, src);
14267     cmovI_reg_imm0_gt(dst, src, cr);
14268   %}
14269 %}
14270 
14271 // This pattern is automatically generated from aarch64_ad.m4
14272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14273 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14274 %{
14275   match(Set dst (MaxI src imm));
14276   ins_cost(INSN_COST * 3);
14277   expand %{
14278     rFlagsReg cr;
14279     compI_reg_imm0(cr, src);
14280     cmovI_reg_imm1_gt(dst, src, cr);
14281   %}
14282 %}
14283 
14284 // This pattern is automatically generated from aarch64_ad.m4
14285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14286 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14287 %{
14288   match(Set dst (MaxI imm src));
14289   ins_cost(INSN_COST * 3);
14290   expand %{
14291     rFlagsReg cr;
14292     compI_reg_imm0(cr, src);
14293     cmovI_reg_imm1_gt(dst, src, cr);
14294   %}
14295 %}
14296 
14297 // This pattern is automatically generated from aarch64_ad.m4
14298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14299 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14300 %{
14301   match(Set dst (MaxI src imm));
14302   ins_cost(INSN_COST * 3);
14303   expand %{
14304     rFlagsReg cr;
14305     compI_reg_imm0(cr, src);
14306     cmovI_reg_immM1_ge(dst, src, cr);
14307   %}
14308 %}
14309 
14310 // This pattern is automatically generated from aarch64_ad.m4
14311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14312 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14313 %{
14314   match(Set dst (MaxI imm src));
14315   ins_cost(INSN_COST * 3);
14316   expand %{
14317     rFlagsReg cr;
14318     compI_reg_imm0(cr, src);
14319     cmovI_reg_immM1_ge(dst, src, cr);
14320   %}
14321 %}
14322 
14323 
14324 
14325 // END This section of the file is automatically generated. Do not edit --------------
14326 
14327 
14328 // ============================================================================
14329 // Floating Point Arithmetic Instructions
14330 
14331 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14332   match(Set dst (AddF src1 src2));
14333 
14334   ins_cost(INSN_COST * 5);
14335   format %{ "fadds   $dst, $src1, $src2" %}
14336 
14337   ins_encode %{
14338     __ fadds(as_FloatRegister($dst$$reg),
14339              as_FloatRegister($src1$$reg),
14340              as_FloatRegister($src2$$reg));
14341   %}
14342 
14343   ins_pipe(fp_dop_reg_reg_s);
14344 %}
14345 
14346 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14347   match(Set dst (AddD src1 src2));
14348 
14349   ins_cost(INSN_COST * 5);
14350   format %{ "faddd   $dst, $src1, $src2" %}
14351 
14352   ins_encode %{
14353     __ faddd(as_FloatRegister($dst$$reg),
14354              as_FloatRegister($src1$$reg),
14355              as_FloatRegister($src2$$reg));
14356   %}
14357 
14358   ins_pipe(fp_dop_reg_reg_d);
14359 %}
14360 
14361 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14362   match(Set dst (SubF src1 src2));
14363 
14364   ins_cost(INSN_COST * 5);
14365   format %{ "fsubs   $dst, $src1, $src2" %}
14366 
14367   ins_encode %{
14368     __ fsubs(as_FloatRegister($dst$$reg),
14369              as_FloatRegister($src1$$reg),
14370              as_FloatRegister($src2$$reg));
14371   %}
14372 
14373   ins_pipe(fp_dop_reg_reg_s);
14374 %}
14375 
14376 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14377   match(Set dst (SubD src1 src2));
14378 
14379   ins_cost(INSN_COST * 5);
14380   format %{ "fsubd   $dst, $src1, $src2" %}
14381 
14382   ins_encode %{
14383     __ fsubd(as_FloatRegister($dst$$reg),
14384              as_FloatRegister($src1$$reg),
14385              as_FloatRegister($src2$$reg));
14386   %}
14387 
14388   ins_pipe(fp_dop_reg_reg_d);
14389 %}
14390 
14391 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14392   match(Set dst (MulF src1 src2));
14393 
14394   ins_cost(INSN_COST * 6);
14395   format %{ "fmuls   $dst, $src1, $src2" %}
14396 
14397   ins_encode %{
14398     __ fmuls(as_FloatRegister($dst$$reg),
14399              as_FloatRegister($src1$$reg),
14400              as_FloatRegister($src2$$reg));
14401   %}
14402 
14403   ins_pipe(fp_dop_reg_reg_s);
14404 %}
14405 
14406 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14407   match(Set dst (MulD src1 src2));
14408 
14409   ins_cost(INSN_COST * 6);
14410   format %{ "fmuld   $dst, $src1, $src2" %}
14411 
14412   ins_encode %{
14413     __ fmuld(as_FloatRegister($dst$$reg),
14414              as_FloatRegister($src1$$reg),
14415              as_FloatRegister($src2$$reg));
14416   %}
14417 
14418   ins_pipe(fp_dop_reg_reg_d);
14419 %}
14420 
14421 // src1 * src2 + src3
14422 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14423   predicate(UseFMA);
14424   match(Set dst (FmaF src3 (Binary src1 src2)));
14425 
14426   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14427 
14428   ins_encode %{
14429     __ fmadds(as_FloatRegister($dst$$reg),
14430              as_FloatRegister($src1$$reg),
14431              as_FloatRegister($src2$$reg),
14432              as_FloatRegister($src3$$reg));
14433   %}
14434 
14435   ins_pipe(pipe_class_default);
14436 %}
14437 
14438 // src1 * src2 + src3
14439 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14440   predicate(UseFMA);
14441   match(Set dst (FmaD src3 (Binary src1 src2)));
14442 
14443   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14444 
14445   ins_encode %{
14446     __ fmaddd(as_FloatRegister($dst$$reg),
14447              as_FloatRegister($src1$$reg),
14448              as_FloatRegister($src2$$reg),
14449              as_FloatRegister($src3$$reg));
14450   %}
14451 
14452   ins_pipe(pipe_class_default);
14453 %}
14454 
14455 // -src1 * src2 + src3
14456 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14457   predicate(UseFMA);
14458   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14459   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14460 
14461   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14462 
14463   ins_encode %{
14464     __ fmsubs(as_FloatRegister($dst$$reg),
14465               as_FloatRegister($src1$$reg),
14466               as_FloatRegister($src2$$reg),
14467               as_FloatRegister($src3$$reg));
14468   %}
14469 
14470   ins_pipe(pipe_class_default);
14471 %}
14472 
14473 // -src1 * src2 + src3
14474 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14475   predicate(UseFMA);
14476   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14477   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14478 
14479   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14480 
14481   ins_encode %{
14482     __ fmsubd(as_FloatRegister($dst$$reg),
14483               as_FloatRegister($src1$$reg),
14484               as_FloatRegister($src2$$reg),
14485               as_FloatRegister($src3$$reg));
14486   %}
14487 
14488   ins_pipe(pipe_class_default);
14489 %}
14490 
14491 // -src1 * src2 - src3
14492 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14493   predicate(UseFMA);
14494   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14495   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14496 
14497   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14498 
14499   ins_encode %{
14500     __ fnmadds(as_FloatRegister($dst$$reg),
14501                as_FloatRegister($src1$$reg),
14502                as_FloatRegister($src2$$reg),
14503                as_FloatRegister($src3$$reg));
14504   %}
14505 
14506   ins_pipe(pipe_class_default);
14507 %}
14508 
14509 // -src1 * src2 - src3
14510 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14511   predicate(UseFMA);
14512   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14513   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14514 
14515   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14516 
14517   ins_encode %{
14518     __ fnmaddd(as_FloatRegister($dst$$reg),
14519                as_FloatRegister($src1$$reg),
14520                as_FloatRegister($src2$$reg),
14521                as_FloatRegister($src3$$reg));
14522   %}
14523 
14524   ins_pipe(pipe_class_default);
14525 %}
14526 
14527 // src1 * src2 - src3
14528 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14529   predicate(UseFMA);
14530   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14531 
14532   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14533 
14534   ins_encode %{
14535     __ fnmsubs(as_FloatRegister($dst$$reg),
14536                as_FloatRegister($src1$$reg),
14537                as_FloatRegister($src2$$reg),
14538                as_FloatRegister($src3$$reg));
14539   %}
14540 
14541   ins_pipe(pipe_class_default);
14542 %}
14543 
14544 // src1 * src2 - src3
14545 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14546   predicate(UseFMA);
14547   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14548 
14549   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14550 
14551   ins_encode %{
14552   // n.b. insn name should be fnmsubd
14553     __ fnmsub(as_FloatRegister($dst$$reg),
14554               as_FloatRegister($src1$$reg),
14555               as_FloatRegister($src2$$reg),
14556               as_FloatRegister($src3$$reg));
14557   %}
14558 
14559   ins_pipe(pipe_class_default);
14560 %}
14561 
14562 
14563 // Math.max(FF)F
14564 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14565   match(Set dst (MaxF src1 src2));
14566 
14567   format %{ "fmaxs   $dst, $src1, $src2" %}
14568   ins_encode %{
14569     __ fmaxs(as_FloatRegister($dst$$reg),
14570              as_FloatRegister($src1$$reg),
14571              as_FloatRegister($src2$$reg));
14572   %}
14573 
14574   ins_pipe(fp_dop_reg_reg_s);
14575 %}
14576 
14577 // Math.min(FF)F
14578 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14579   match(Set dst (MinF src1 src2));
14580 
14581   format %{ "fmins   $dst, $src1, $src2" %}
14582   ins_encode %{
14583     __ fmins(as_FloatRegister($dst$$reg),
14584              as_FloatRegister($src1$$reg),
14585              as_FloatRegister($src2$$reg));
14586   %}
14587 
14588   ins_pipe(fp_dop_reg_reg_s);
14589 %}
14590 
14591 // Math.max(DD)D
14592 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14593   match(Set dst (MaxD src1 src2));
14594 
14595   format %{ "fmaxd   $dst, $src1, $src2" %}
14596   ins_encode %{
14597     __ fmaxd(as_FloatRegister($dst$$reg),
14598              as_FloatRegister($src1$$reg),
14599              as_FloatRegister($src2$$reg));
14600   %}
14601 
14602   ins_pipe(fp_dop_reg_reg_d);
14603 %}
14604 
14605 // Math.min(DD)D
14606 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14607   match(Set dst (MinD src1 src2));
14608 
14609   format %{ "fmind   $dst, $src1, $src2" %}
14610   ins_encode %{
14611     __ fmind(as_FloatRegister($dst$$reg),
14612              as_FloatRegister($src1$$reg),
14613              as_FloatRegister($src2$$reg));
14614   %}
14615 
14616   ins_pipe(fp_dop_reg_reg_d);
14617 %}
14618 
14619 
14620 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14621   match(Set dst (DivF src1  src2));
14622 
14623   ins_cost(INSN_COST * 18);
14624   format %{ "fdivs   $dst, $src1, $src2" %}
14625 
14626   ins_encode %{
14627     __ fdivs(as_FloatRegister($dst$$reg),
14628              as_FloatRegister($src1$$reg),
14629              as_FloatRegister($src2$$reg));
14630   %}
14631 
14632   ins_pipe(fp_div_s);
14633 %}
14634 
14635 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14636   match(Set dst (DivD src1  src2));
14637 
14638   ins_cost(INSN_COST * 32);
14639   format %{ "fdivd   $dst, $src1, $src2" %}
14640 
14641   ins_encode %{
14642     __ fdivd(as_FloatRegister($dst$$reg),
14643              as_FloatRegister($src1$$reg),
14644              as_FloatRegister($src2$$reg));
14645   %}
14646 
14647   ins_pipe(fp_div_d);
14648 %}
14649 
14650 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14651   match(Set dst (NegF src));
14652 
14653   ins_cost(INSN_COST * 3);
14654   format %{ "fneg   $dst, $src" %}
14655 
14656   ins_encode %{
14657     __ fnegs(as_FloatRegister($dst$$reg),
14658              as_FloatRegister($src$$reg));
14659   %}
14660 
14661   ins_pipe(fp_uop_s);
14662 %}
14663 
14664 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14665   match(Set dst (NegD src));
14666 
14667   ins_cost(INSN_COST * 3);
14668   format %{ "fnegd   $dst, $src" %}
14669 
14670   ins_encode %{
14671     __ fnegd(as_FloatRegister($dst$$reg),
14672              as_FloatRegister($src$$reg));
14673   %}
14674 
14675   ins_pipe(fp_uop_d);
14676 %}
14677 
14678 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14679 %{
14680   match(Set dst (AbsI src));
14681 
14682   effect(KILL cr);
14683   ins_cost(INSN_COST * 2);
14684   format %{ "cmpw  $src, zr\n\t"
14685             "cnegw $dst, $src, Assembler::LT\t# int abs"
14686   %}
14687 
14688   ins_encode %{
14689     __ cmpw(as_Register($src$$reg), zr);
14690     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14691   %}
14692   ins_pipe(pipe_class_default);
14693 %}
14694 
14695 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14696 %{
14697   match(Set dst (AbsL src));
14698 
14699   effect(KILL cr);
14700   ins_cost(INSN_COST * 2);
14701   format %{ "cmp  $src, zr\n\t"
14702             "cneg $dst, $src, Assembler::LT\t# long abs"
14703   %}
14704 
14705   ins_encode %{
14706     __ cmp(as_Register($src$$reg), zr);
14707     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14708   %}
14709   ins_pipe(pipe_class_default);
14710 %}
14711 
14712 instruct absF_reg(vRegF dst, vRegF src) %{
14713   match(Set dst (AbsF src));
14714 
14715   ins_cost(INSN_COST * 3);
14716   format %{ "fabss   $dst, $src" %}
14717   ins_encode %{
14718     __ fabss(as_FloatRegister($dst$$reg),
14719              as_FloatRegister($src$$reg));
14720   %}
14721 
14722   ins_pipe(fp_uop_s);
14723 %}
14724 
14725 instruct absD_reg(vRegD dst, vRegD src) %{
14726   match(Set dst (AbsD src));
14727 
14728   ins_cost(INSN_COST * 3);
14729   format %{ "fabsd   $dst, $src" %}
14730   ins_encode %{
14731     __ fabsd(as_FloatRegister($dst$$reg),
14732              as_FloatRegister($src$$reg));
14733   %}
14734 
14735   ins_pipe(fp_uop_d);
14736 %}
14737 
14738 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14739   match(Set dst (AbsF (SubF src1 src2)));
14740 
14741   ins_cost(INSN_COST * 3);
14742   format %{ "fabds   $dst, $src1, $src2" %}
14743   ins_encode %{
14744     __ fabds(as_FloatRegister($dst$$reg),
14745              as_FloatRegister($src1$$reg),
14746              as_FloatRegister($src2$$reg));
14747   %}
14748 
14749   ins_pipe(fp_uop_s);
14750 %}
14751 
14752 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14753   match(Set dst (AbsD (SubD src1 src2)));
14754 
14755   ins_cost(INSN_COST * 3);
14756   format %{ "fabdd   $dst, $src1, $src2" %}
14757   ins_encode %{
14758     __ fabdd(as_FloatRegister($dst$$reg),
14759              as_FloatRegister($src1$$reg),
14760              as_FloatRegister($src2$$reg));
14761   %}
14762 
14763   ins_pipe(fp_uop_d);
14764 %}
14765 
14766 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14767   match(Set dst (SqrtD src));
14768 
14769   ins_cost(INSN_COST * 50);
14770   format %{ "fsqrtd  $dst, $src" %}
14771   ins_encode %{
14772     __ fsqrtd(as_FloatRegister($dst$$reg),
14773              as_FloatRegister($src$$reg));
14774   %}
14775 
14776   ins_pipe(fp_div_s);
14777 %}
14778 
14779 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14780   match(Set dst (SqrtF src));
14781 
14782   ins_cost(INSN_COST * 50);
14783   format %{ "fsqrts  $dst, $src" %}
14784   ins_encode %{
14785     __ fsqrts(as_FloatRegister($dst$$reg),
14786              as_FloatRegister($src$$reg));
14787   %}
14788 
14789   ins_pipe(fp_div_d);
14790 %}
14791 
14792 // Math.rint, floor, ceil
14793 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14794   match(Set dst (RoundDoubleMode src rmode));
14795   format %{ "frint  $dst, $src, $rmode" %}
14796   ins_encode %{
14797     switch ($rmode$$constant) {
14798       case RoundDoubleModeNode::rmode_rint:
14799         __ frintnd(as_FloatRegister($dst$$reg),
14800                    as_FloatRegister($src$$reg));
14801         break;
14802       case RoundDoubleModeNode::rmode_floor:
14803         __ frintmd(as_FloatRegister($dst$$reg),
14804                    as_FloatRegister($src$$reg));
14805         break;
14806       case RoundDoubleModeNode::rmode_ceil:
14807         __ frintpd(as_FloatRegister($dst$$reg),
14808                    as_FloatRegister($src$$reg));
14809         break;
14810     }
14811   %}
14812   ins_pipe(fp_uop_d);
14813 %}
14814 
14815 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14816   match(Set dst (CopySignD src1 (Binary src2 zero)));
14817   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14818   format %{ "CopySignD  $dst $src1 $src2" %}
14819   ins_encode %{
14820     FloatRegister dst = as_FloatRegister($dst$$reg),
14821                   src1 = as_FloatRegister($src1$$reg),
14822                   src2 = as_FloatRegister($src2$$reg),
14823                   zero = as_FloatRegister($zero$$reg);
14824     __ fnegd(dst, zero);
14825     __ bsl(dst, __ T8B, src2, src1);
14826   %}
14827   ins_pipe(fp_uop_d);
14828 %}
14829 
14830 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14831   match(Set dst (CopySignF src1 src2));
14832   effect(TEMP_DEF dst, USE src1, USE src2);
14833   format %{ "CopySignF  $dst $src1 $src2" %}
14834   ins_encode %{
14835     FloatRegister dst = as_FloatRegister($dst$$reg),
14836                   src1 = as_FloatRegister($src1$$reg),
14837                   src2 = as_FloatRegister($src2$$reg);
14838     __ movi(dst, __ T2S, 0x80, 24);
14839     __ bsl(dst, __ T8B, src2, src1);
14840   %}
14841   ins_pipe(fp_uop_d);
14842 %}
14843 
14844 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14845   match(Set dst (SignumD src (Binary zero one)));
14846   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14847   format %{ "signumD  $dst, $src" %}
14848   ins_encode %{
14849     FloatRegister src = as_FloatRegister($src$$reg),
14850                   dst = as_FloatRegister($dst$$reg),
14851                   zero = as_FloatRegister($zero$$reg),
14852                   one = as_FloatRegister($one$$reg);
14853     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14854     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14855     // Bit selection instruction gets bit from "one" for each enabled bit in
14856     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14857     // NaN the whole "src" will be copied because "dst" is zero. For all other
14858     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14859     // from "src", and all other bits are copied from 1.0.
14860     __ bsl(dst, __ T8B, one, src);
14861   %}
14862   ins_pipe(fp_uop_d);
14863 %}
14864 
14865 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14866   match(Set dst (SignumF src (Binary zero one)));
14867   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14868   format %{ "signumF  $dst, $src" %}
14869   ins_encode %{
14870     FloatRegister src = as_FloatRegister($src$$reg),
14871                   dst = as_FloatRegister($dst$$reg),
14872                   zero = as_FloatRegister($zero$$reg),
14873                   one = as_FloatRegister($one$$reg);
14874     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14875     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14876     // Bit selection instruction gets bit from "one" for each enabled bit in
14877     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14878     // NaN the whole "src" will be copied because "dst" is zero. For all other
14879     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14880     // from "src", and all other bits are copied from 1.0.
14881     __ bsl(dst, __ T8B, one, src);
14882   %}
14883   ins_pipe(fp_uop_d);
14884 %}
14885 
14886 instruct onspinwait() %{
14887   match(OnSpinWait);
14888   ins_cost(INSN_COST);
14889 
14890   format %{ "onspinwait" %}
14891 
14892   ins_encode %{
14893     __ spin_wait();
14894   %}
14895   ins_pipe(pipe_class_empty);
14896 %}
14897 
14898 // ============================================================================
14899 // Logical Instructions
14900 
14901 // Integer Logical Instructions
14902 
14903 // And Instructions
14904 
14905 
14906 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14907   match(Set dst (AndI src1 src2));
14908 
14909   format %{ "andw  $dst, $src1, $src2\t# int" %}
14910 
14911   ins_cost(INSN_COST);
14912   ins_encode %{
14913     __ andw(as_Register($dst$$reg),
14914             as_Register($src1$$reg),
14915             as_Register($src2$$reg));
14916   %}
14917 
14918   ins_pipe(ialu_reg_reg);
14919 %}
14920 
14921 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14922   match(Set dst (AndI src1 src2));
14923 
14924   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14925 
14926   ins_cost(INSN_COST);
14927   ins_encode %{
14928     __ andw(as_Register($dst$$reg),
14929             as_Register($src1$$reg),
14930             (uint64_t)($src2$$constant));
14931   %}
14932 
14933   ins_pipe(ialu_reg_imm);
14934 %}
14935 
14936 // Or Instructions
14937 
14938 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14939   match(Set dst (OrI src1 src2));
14940 
14941   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14942 
14943   ins_cost(INSN_COST);
14944   ins_encode %{
14945     __ orrw(as_Register($dst$$reg),
14946             as_Register($src1$$reg),
14947             as_Register($src2$$reg));
14948   %}
14949 
14950   ins_pipe(ialu_reg_reg);
14951 %}
14952 
14953 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14954   match(Set dst (OrI src1 src2));
14955 
14956   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14957 
14958   ins_cost(INSN_COST);
14959   ins_encode %{
14960     __ orrw(as_Register($dst$$reg),
14961             as_Register($src1$$reg),
14962             (uint64_t)($src2$$constant));
14963   %}
14964 
14965   ins_pipe(ialu_reg_imm);
14966 %}
14967 
14968 // Xor Instructions
14969 
14970 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14971   match(Set dst (XorI src1 src2));
14972 
14973   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14974 
14975   ins_cost(INSN_COST);
14976   ins_encode %{
14977     __ eorw(as_Register($dst$$reg),
14978             as_Register($src1$$reg),
14979             as_Register($src2$$reg));
14980   %}
14981 
14982   ins_pipe(ialu_reg_reg);
14983 %}
14984 
14985 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14986   match(Set dst (XorI src1 src2));
14987 
14988   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14989 
14990   ins_cost(INSN_COST);
14991   ins_encode %{
14992     __ eorw(as_Register($dst$$reg),
14993             as_Register($src1$$reg),
14994             (uint64_t)($src2$$constant));
14995   %}
14996 
14997   ins_pipe(ialu_reg_imm);
14998 %}
14999 
15000 // Long Logical Instructions
15001 // TODO
15002 
15003 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
15004   match(Set dst (AndL src1 src2));
15005 
15006   format %{ "and  $dst, $src1, $src2\t# int" %}
15007 
15008   ins_cost(INSN_COST);
15009   ins_encode %{
15010     __ andr(as_Register($dst$$reg),
15011             as_Register($src1$$reg),
15012             as_Register($src2$$reg));
15013   %}
15014 
15015   ins_pipe(ialu_reg_reg);
15016 %}
15017 
15018 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
15019   match(Set dst (AndL src1 src2));
15020 
15021   format %{ "and  $dst, $src1, $src2\t# int" %}
15022 
15023   ins_cost(INSN_COST);
15024   ins_encode %{
15025     __ andr(as_Register($dst$$reg),
15026             as_Register($src1$$reg),
15027             (uint64_t)($src2$$constant));
15028   %}
15029 
15030   ins_pipe(ialu_reg_imm);
15031 %}
15032 
15033 // Or Instructions
15034 
15035 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
15036   match(Set dst (OrL src1 src2));
15037 
15038   format %{ "orr  $dst, $src1, $src2\t# int" %}
15039 
15040   ins_cost(INSN_COST);
15041   ins_encode %{
15042     __ orr(as_Register($dst$$reg),
15043            as_Register($src1$$reg),
15044            as_Register($src2$$reg));
15045   %}
15046 
15047   ins_pipe(ialu_reg_reg);
15048 %}
15049 
15050 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
15051   match(Set dst (OrL src1 src2));
15052 
15053   format %{ "orr  $dst, $src1, $src2\t# int" %}
15054 
15055   ins_cost(INSN_COST);
15056   ins_encode %{
15057     __ orr(as_Register($dst$$reg),
15058            as_Register($src1$$reg),
15059            (uint64_t)($src2$$constant));
15060   %}
15061 
15062   ins_pipe(ialu_reg_imm);
15063 %}
15064 
15065 // Xor Instructions
15066 
15067 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
15068   match(Set dst (XorL src1 src2));
15069 
15070   format %{ "eor  $dst, $src1, $src2\t# int" %}
15071 
15072   ins_cost(INSN_COST);
15073   ins_encode %{
15074     __ eor(as_Register($dst$$reg),
15075            as_Register($src1$$reg),
15076            as_Register($src2$$reg));
15077   %}
15078 
15079   ins_pipe(ialu_reg_reg);
15080 %}
15081 
15082 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
15083   match(Set dst (XorL src1 src2));
15084 
15085   ins_cost(INSN_COST);
15086   format %{ "eor  $dst, $src1, $src2\t# int" %}
15087 
15088   ins_encode %{
15089     __ eor(as_Register($dst$$reg),
15090            as_Register($src1$$reg),
15091            (uint64_t)($src2$$constant));
15092   %}
15093 
15094   ins_pipe(ialu_reg_imm);
15095 %}
15096 
15097 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
15098 %{
15099   match(Set dst (ConvI2L src));
15100 
15101   ins_cost(INSN_COST);
15102   format %{ "sxtw  $dst, $src\t# i2l" %}
15103   ins_encode %{
15104     __ sbfm($dst$$Register, $src$$Register, 0, 31);
15105   %}
15106   ins_pipe(ialu_reg_shift);
15107 %}
15108 
15109 // this pattern occurs in bigmath arithmetic
15110 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
15111 %{
15112   match(Set dst (AndL (ConvI2L src) mask));
15113 
15114   ins_cost(INSN_COST);
15115   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
15116   ins_encode %{
15117     __ ubfm($dst$$Register, $src$$Register, 0, 31);
15118   %}
15119 
15120   ins_pipe(ialu_reg_shift);
15121 %}
15122 
15123 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
15124   match(Set dst (ConvL2I src));
15125 
15126   ins_cost(INSN_COST);
15127   format %{ "movw  $dst, $src \t// l2i" %}
15128 
15129   ins_encode %{
15130     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
15131   %}
15132 
15133   ins_pipe(ialu_reg);
15134 %}
15135 
15136 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
15137 %{
15138   match(Set dst (Conv2B src));
15139   effect(KILL cr);
15140 
15141   format %{
15142     "cmpw $src, zr\n\t"
15143     "cset $dst, ne"
15144   %}
15145 
15146   ins_encode %{
15147     __ cmpw(as_Register($src$$reg), zr);
15148     __ cset(as_Register($dst$$reg), Assembler::NE);
15149   %}
15150 
15151   ins_pipe(ialu_reg);
15152 %}
15153 
15154 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
15155 %{
15156   match(Set dst (Conv2B src));
15157   effect(KILL cr);
15158 
15159   format %{
15160     "cmp  $src, zr\n\t"
15161     "cset $dst, ne"
15162   %}
15163 
15164   ins_encode %{
15165     __ cmp(as_Register($src$$reg), zr);
15166     __ cset(as_Register($dst$$reg), Assembler::NE);
15167   %}
15168 
15169   ins_pipe(ialu_reg);
15170 %}
15171 
15172 instruct convD2F_reg(vRegF dst, vRegD src) %{
15173   match(Set dst (ConvD2F src));
15174 
15175   ins_cost(INSN_COST * 5);
15176   format %{ "fcvtd  $dst, $src \t// d2f" %}
15177 
15178   ins_encode %{
15179     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15180   %}
15181 
15182   ins_pipe(fp_d2f);
15183 %}
15184 
15185 instruct convF2D_reg(vRegD dst, vRegF src) %{
15186   match(Set dst (ConvF2D src));
15187 
15188   ins_cost(INSN_COST * 5);
15189   format %{ "fcvts  $dst, $src \t// f2d" %}
15190 
15191   ins_encode %{
15192     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15193   %}
15194 
15195   ins_pipe(fp_f2d);
15196 %}
15197 
15198 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15199   match(Set dst (ConvF2I src));
15200 
15201   ins_cost(INSN_COST * 5);
15202   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
15203 
15204   ins_encode %{
15205     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15206   %}
15207 
15208   ins_pipe(fp_f2i);
15209 %}
15210 
15211 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15212   match(Set dst (ConvF2L src));
15213 
15214   ins_cost(INSN_COST * 5);
15215   format %{ "fcvtzs  $dst, $src \t// f2l" %}
15216 
15217   ins_encode %{
15218     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15219   %}
15220 
15221   ins_pipe(fp_f2l);
15222 %}
15223 
15224 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15225   match(Set dst (ConvI2F src));
15226 
15227   ins_cost(INSN_COST * 5);
15228   format %{ "scvtfws  $dst, $src \t// i2f" %}
15229 
15230   ins_encode %{
15231     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15232   %}
15233 
15234   ins_pipe(fp_i2f);
15235 %}
15236 
15237 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15238   match(Set dst (ConvL2F src));
15239 
15240   ins_cost(INSN_COST * 5);
15241   format %{ "scvtfs  $dst, $src \t// l2f" %}
15242 
15243   ins_encode %{
15244     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15245   %}
15246 
15247   ins_pipe(fp_l2f);
15248 %}
15249 
15250 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15251   match(Set dst (ConvD2I src));
15252 
15253   ins_cost(INSN_COST * 5);
15254   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15255 
15256   ins_encode %{
15257     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15258   %}
15259 
15260   ins_pipe(fp_d2i);
15261 %}
15262 
15263 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15264   match(Set dst (ConvD2L src));
15265 
15266   ins_cost(INSN_COST * 5);
15267   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15268 
15269   ins_encode %{
15270     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15271   %}
15272 
15273   ins_pipe(fp_d2l);
15274 %}
15275 
15276 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15277   match(Set dst (ConvI2D src));
15278 
15279   ins_cost(INSN_COST * 5);
15280   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15281 
15282   ins_encode %{
15283     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15284   %}
15285 
15286   ins_pipe(fp_i2d);
15287 %}
15288 
15289 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15290   match(Set dst (ConvL2D src));
15291 
15292   ins_cost(INSN_COST * 5);
15293   format %{ "scvtfd  $dst, $src \t// l2d" %}
15294 
15295   ins_encode %{
15296     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15297   %}
15298 
15299   ins_pipe(fp_l2d);
15300 %}
15301 
15302 // stack <-> reg and reg <-> reg shuffles with no conversion
15303 
15304 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15305 
15306   match(Set dst (MoveF2I src));
15307 
15308   effect(DEF dst, USE src);
15309 
15310   ins_cost(4 * INSN_COST);
15311 
15312   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15313 
15314   ins_encode %{
15315     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15316   %}
15317 
15318   ins_pipe(iload_reg_reg);
15319 
15320 %}
15321 
15322 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15323 
15324   match(Set dst (MoveI2F src));
15325 
15326   effect(DEF dst, USE src);
15327 
15328   ins_cost(4 * INSN_COST);
15329 
15330   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15331 
15332   ins_encode %{
15333     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15334   %}
15335 
15336   ins_pipe(pipe_class_memory);
15337 
15338 %}
15339 
15340 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15341 
15342   match(Set dst (MoveD2L src));
15343 
15344   effect(DEF dst, USE src);
15345 
15346   ins_cost(4 * INSN_COST);
15347 
15348   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15349 
15350   ins_encode %{
15351     __ ldr($dst$$Register, Address(sp, $src$$disp));
15352   %}
15353 
15354   ins_pipe(iload_reg_reg);
15355 
15356 %}
15357 
15358 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15359 
15360   match(Set dst (MoveL2D src));
15361 
15362   effect(DEF dst, USE src);
15363 
15364   ins_cost(4 * INSN_COST);
15365 
15366   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15367 
15368   ins_encode %{
15369     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15370   %}
15371 
15372   ins_pipe(pipe_class_memory);
15373 
15374 %}
15375 
15376 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15377 
15378   match(Set dst (MoveF2I src));
15379 
15380   effect(DEF dst, USE src);
15381 
15382   ins_cost(INSN_COST);
15383 
15384   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15385 
15386   ins_encode %{
15387     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15388   %}
15389 
15390   ins_pipe(pipe_class_memory);
15391 
15392 %}
15393 
15394 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15395 
15396   match(Set dst (MoveI2F src));
15397 
15398   effect(DEF dst, USE src);
15399 
15400   ins_cost(INSN_COST);
15401 
15402   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15403 
15404   ins_encode %{
15405     __ strw($src$$Register, Address(sp, $dst$$disp));
15406   %}
15407 
15408   ins_pipe(istore_reg_reg);
15409 
15410 %}
15411 
15412 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15413 
15414   match(Set dst (MoveD2L src));
15415 
15416   effect(DEF dst, USE src);
15417 
15418   ins_cost(INSN_COST);
15419 
15420   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15421 
15422   ins_encode %{
15423     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15424   %}
15425 
15426   ins_pipe(pipe_class_memory);
15427 
15428 %}
15429 
15430 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15431 
15432   match(Set dst (MoveL2D src));
15433 
15434   effect(DEF dst, USE src);
15435 
15436   ins_cost(INSN_COST);
15437 
15438   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15439 
15440   ins_encode %{
15441     __ str($src$$Register, Address(sp, $dst$$disp));
15442   %}
15443 
15444   ins_pipe(istore_reg_reg);
15445 
15446 %}
15447 
15448 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15449 
15450   match(Set dst (MoveF2I src));
15451 
15452   effect(DEF dst, USE src);
15453 
15454   ins_cost(INSN_COST);
15455 
15456   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15457 
15458   ins_encode %{
15459     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15460   %}
15461 
15462   ins_pipe(fp_f2i);
15463 
15464 %}
15465 
15466 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15467 
15468   match(Set dst (MoveI2F src));
15469 
15470   effect(DEF dst, USE src);
15471 
15472   ins_cost(INSN_COST);
15473 
15474   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15475 
15476   ins_encode %{
15477     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15478   %}
15479 
15480   ins_pipe(fp_i2f);
15481 
15482 %}
15483 
15484 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15485 
15486   match(Set dst (MoveD2L src));
15487 
15488   effect(DEF dst, USE src);
15489 
15490   ins_cost(INSN_COST);
15491 
15492   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15493 
15494   ins_encode %{
15495     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15496   %}
15497 
15498   ins_pipe(fp_d2l);
15499 
15500 %}
15501 
15502 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15503 
15504   match(Set dst (MoveL2D src));
15505 
15506   effect(DEF dst, USE src);
15507 
15508   ins_cost(INSN_COST);
15509 
15510   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15511 
15512   ins_encode %{
15513     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15514   %}
15515 
15516   ins_pipe(fp_l2d);
15517 
15518 %}
15519 
15520 // ============================================================================
15521 // clearing of an array
15522 
15523 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15524 %{
15525   match(Set dummy (ClearArray cnt base));
15526   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15527 
15528   ins_cost(4 * INSN_COST);
15529   format %{ "ClearArray $cnt, $base" %}
15530 
15531   ins_encode %{
15532     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15533     if (tpc == NULL) {
15534       ciEnv::current()->record_failure("CodeCache is full");
15535       return;
15536     }
15537   %}
15538 
15539   ins_pipe(pipe_class_memory);
15540 %}
15541 
15542 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15543 %{
15544   predicate((uint64_t)n->in(2)->get_long()
15545             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15546   match(Set dummy (ClearArray cnt base));
15547   effect(TEMP temp, USE_KILL base, KILL cr);
15548 
15549   ins_cost(4 * INSN_COST);
15550   format %{ "ClearArray $cnt, $base" %}
15551 
15552   ins_encode %{
15553     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15554     if (tpc == NULL) {
15555       ciEnv::current()->record_failure("CodeCache is full");
15556       return;
15557     }
15558   %}
15559 
15560   ins_pipe(pipe_class_memory);
15561 %}
15562 
15563 // ============================================================================
15564 // Overflow Math Instructions
15565 
15566 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15567 %{
15568   match(Set cr (OverflowAddI op1 op2));
15569 
15570   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15571   ins_cost(INSN_COST);
15572   ins_encode %{
15573     __ cmnw($op1$$Register, $op2$$Register);
15574   %}
15575 
15576   ins_pipe(icmp_reg_reg);
15577 %}
15578 
15579 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15580 %{
15581   match(Set cr (OverflowAddI op1 op2));
15582 
15583   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15584   ins_cost(INSN_COST);
15585   ins_encode %{
15586     __ cmnw($op1$$Register, $op2$$constant);
15587   %}
15588 
15589   ins_pipe(icmp_reg_imm);
15590 %}
15591 
15592 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15593 %{
15594   match(Set cr (OverflowAddL op1 op2));
15595 
15596   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15597   ins_cost(INSN_COST);
15598   ins_encode %{
15599     __ cmn($op1$$Register, $op2$$Register);
15600   %}
15601 
15602   ins_pipe(icmp_reg_reg);
15603 %}
15604 
15605 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15606 %{
15607   match(Set cr (OverflowAddL op1 op2));
15608 
15609   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15610   ins_cost(INSN_COST);
15611   ins_encode %{
15612     __ adds(zr, $op1$$Register, $op2$$constant);
15613   %}
15614 
15615   ins_pipe(icmp_reg_imm);
15616 %}
15617 
15618 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15619 %{
15620   match(Set cr (OverflowSubI op1 op2));
15621 
15622   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15623   ins_cost(INSN_COST);
15624   ins_encode %{
15625     __ cmpw($op1$$Register, $op2$$Register);
15626   %}
15627 
15628   ins_pipe(icmp_reg_reg);
15629 %}
15630 
15631 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15632 %{
15633   match(Set cr (OverflowSubI op1 op2));
15634 
15635   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15636   ins_cost(INSN_COST);
15637   ins_encode %{
15638     __ cmpw($op1$$Register, $op2$$constant);
15639   %}
15640 
15641   ins_pipe(icmp_reg_imm);
15642 %}
15643 
15644 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15645 %{
15646   match(Set cr (OverflowSubL op1 op2));
15647 
15648   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15649   ins_cost(INSN_COST);
15650   ins_encode %{
15651     __ cmp($op1$$Register, $op2$$Register);
15652   %}
15653 
15654   ins_pipe(icmp_reg_reg);
15655 %}
15656 
15657 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15658 %{
15659   match(Set cr (OverflowSubL op1 op2));
15660 
15661   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15662   ins_cost(INSN_COST);
15663   ins_encode %{
15664     __ subs(zr, $op1$$Register, $op2$$constant);
15665   %}
15666 
15667   ins_pipe(icmp_reg_imm);
15668 %}
15669 
15670 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15671 %{
15672   match(Set cr (OverflowSubI zero op1));
15673 
15674   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15675   ins_cost(INSN_COST);
15676   ins_encode %{
15677     __ cmpw(zr, $op1$$Register);
15678   %}
15679 
15680   ins_pipe(icmp_reg_imm);
15681 %}
15682 
15683 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15684 %{
15685   match(Set cr (OverflowSubL zero op1));
15686 
15687   format %{ "cmp   zr, $op1\t# overflow check long" %}
15688   ins_cost(INSN_COST);
15689   ins_encode %{
15690     __ cmp(zr, $op1$$Register);
15691   %}
15692 
15693   ins_pipe(icmp_reg_imm);
15694 %}
15695 
15696 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15697 %{
15698   match(Set cr (OverflowMulI op1 op2));
15699 
15700   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15701             "cmp   rscratch1, rscratch1, sxtw\n\t"
15702             "movw  rscratch1, #0x80000000\n\t"
15703             "cselw rscratch1, rscratch1, zr, NE\n\t"
15704             "cmpw  rscratch1, #1" %}
15705   ins_cost(5 * INSN_COST);
15706   ins_encode %{
15707     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15708     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15709     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15710     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15711     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15712   %}
15713 
15714   ins_pipe(pipe_slow);
15715 %}
15716 
15717 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15718 %{
15719   match(If cmp (OverflowMulI op1 op2));
15720   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15721             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15722   effect(USE labl, KILL cr);
15723 
15724   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15725             "cmp   rscratch1, rscratch1, sxtw\n\t"
15726             "b$cmp   $labl" %}
15727   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15728   ins_encode %{
15729     Label* L = $labl$$label;
15730     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15731     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15732     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15733     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15734   %}
15735 
15736   ins_pipe(pipe_serial);
15737 %}
15738 
15739 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15740 %{
15741   match(Set cr (OverflowMulL op1 op2));
15742 
15743   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15744             "smulh rscratch2, $op1, $op2\n\t"
15745             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15746             "movw  rscratch1, #0x80000000\n\t"
15747             "cselw rscratch1, rscratch1, zr, NE\n\t"
15748             "cmpw  rscratch1, #1" %}
15749   ins_cost(6 * INSN_COST);
15750   ins_encode %{
15751     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15752     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15753     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15754     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15755     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15756     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15757   %}
15758 
15759   ins_pipe(pipe_slow);
15760 %}
15761 
15762 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15763 %{
15764   match(If cmp (OverflowMulL op1 op2));
15765   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15766             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15767   effect(USE labl, KILL cr);
15768 
15769   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15770             "smulh rscratch2, $op1, $op2\n\t"
15771             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15772             "b$cmp $labl" %}
15773   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15774   ins_encode %{
15775     Label* L = $labl$$label;
15776     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15777     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15778     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15779     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15780     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15781   %}
15782 
15783   ins_pipe(pipe_serial);
15784 %}
15785 
15786 // ============================================================================
15787 // Compare Instructions
15788 
15789 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15790 %{
15791   match(Set cr (CmpI op1 op2));
15792 
15793   effect(DEF cr, USE op1, USE op2);
15794 
15795   ins_cost(INSN_COST);
15796   format %{ "cmpw  $op1, $op2" %}
15797 
15798   ins_encode(aarch64_enc_cmpw(op1, op2));
15799 
15800   ins_pipe(icmp_reg_reg);
15801 %}
15802 
15803 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15804 %{
15805   match(Set cr (CmpI op1 zero));
15806 
15807   effect(DEF cr, USE op1);
15808 
15809   ins_cost(INSN_COST);
15810   format %{ "cmpw $op1, 0" %}
15811 
15812   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15813 
15814   ins_pipe(icmp_reg_imm);
15815 %}
15816 
15817 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15818 %{
15819   match(Set cr (CmpI op1 op2));
15820 
15821   effect(DEF cr, USE op1);
15822 
15823   ins_cost(INSN_COST);
15824   format %{ "cmpw  $op1, $op2" %}
15825 
15826   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15827 
15828   ins_pipe(icmp_reg_imm);
15829 %}
15830 
15831 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15832 %{
15833   match(Set cr (CmpI op1 op2));
15834 
15835   effect(DEF cr, USE op1);
15836 
15837   ins_cost(INSN_COST * 2);
15838   format %{ "cmpw  $op1, $op2" %}
15839 
15840   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15841 
15842   ins_pipe(icmp_reg_imm);
15843 %}
15844 
15845 // Unsigned compare Instructions; really, same as signed compare
15846 // except it should only be used to feed an If or a CMovI which takes a
15847 // cmpOpU.
15848 
15849 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15850 %{
15851   match(Set cr (CmpU op1 op2));
15852 
15853   effect(DEF cr, USE op1, USE op2);
15854 
15855   ins_cost(INSN_COST);
15856   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15857 
15858   ins_encode(aarch64_enc_cmpw(op1, op2));
15859 
15860   ins_pipe(icmp_reg_reg);
15861 %}
15862 
15863 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15864 %{
15865   match(Set cr (CmpU op1 zero));
15866 
15867   effect(DEF cr, USE op1);
15868 
15869   ins_cost(INSN_COST);
15870   format %{ "cmpw $op1, #0\t# unsigned" %}
15871 
15872   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15873 
15874   ins_pipe(icmp_reg_imm);
15875 %}
15876 
15877 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15878 %{
15879   match(Set cr (CmpU op1 op2));
15880 
15881   effect(DEF cr, USE op1);
15882 
15883   ins_cost(INSN_COST);
15884   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15885 
15886   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15887 
15888   ins_pipe(icmp_reg_imm);
15889 %}
15890 
15891 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15892 %{
15893   match(Set cr (CmpU op1 op2));
15894 
15895   effect(DEF cr, USE op1);
15896 
15897   ins_cost(INSN_COST * 2);
15898   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15899 
15900   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15901 
15902   ins_pipe(icmp_reg_imm);
15903 %}
15904 
15905 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15906 %{
15907   match(Set cr (CmpL op1 op2));
15908 
15909   effect(DEF cr, USE op1, USE op2);
15910 
15911   ins_cost(INSN_COST);
15912   format %{ "cmp  $op1, $op2" %}
15913 
15914   ins_encode(aarch64_enc_cmp(op1, op2));
15915 
15916   ins_pipe(icmp_reg_reg);
15917 %}
15918 
15919 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15920 %{
15921   match(Set cr (CmpL op1 zero));
15922 
15923   effect(DEF cr, USE op1);
15924 
15925   ins_cost(INSN_COST);
15926   format %{ "tst  $op1" %}
15927 
15928   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15929 
15930   ins_pipe(icmp_reg_imm);
15931 %}
15932 
15933 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15934 %{
15935   match(Set cr (CmpL op1 op2));
15936 
15937   effect(DEF cr, USE op1);
15938 
15939   ins_cost(INSN_COST);
15940   format %{ "cmp  $op1, $op2" %}
15941 
15942   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15943 
15944   ins_pipe(icmp_reg_imm);
15945 %}
15946 
15947 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15948 %{
15949   match(Set cr (CmpL op1 op2));
15950 
15951   effect(DEF cr, USE op1);
15952 
15953   ins_cost(INSN_COST * 2);
15954   format %{ "cmp  $op1, $op2" %}
15955 
15956   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15957 
15958   ins_pipe(icmp_reg_imm);
15959 %}
15960 
15961 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15962 %{
15963   match(Set cr (CmpUL op1 op2));
15964 
15965   effect(DEF cr, USE op1, USE op2);
15966 
15967   ins_cost(INSN_COST);
15968   format %{ "cmp  $op1, $op2" %}
15969 
15970   ins_encode(aarch64_enc_cmp(op1, op2));
15971 
15972   ins_pipe(icmp_reg_reg);
15973 %}
15974 
15975 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15976 %{
15977   match(Set cr (CmpUL op1 zero));
15978 
15979   effect(DEF cr, USE op1);
15980 
15981   ins_cost(INSN_COST);
15982   format %{ "tst  $op1" %}
15983 
15984   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15985 
15986   ins_pipe(icmp_reg_imm);
15987 %}
15988 
15989 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15990 %{
15991   match(Set cr (CmpUL op1 op2));
15992 
15993   effect(DEF cr, USE op1);
15994 
15995   ins_cost(INSN_COST);
15996   format %{ "cmp  $op1, $op2" %}
15997 
15998   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15999 
16000   ins_pipe(icmp_reg_imm);
16001 %}
16002 
16003 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
16004 %{
16005   match(Set cr (CmpUL op1 op2));
16006 
16007   effect(DEF cr, USE op1);
16008 
16009   ins_cost(INSN_COST * 2);
16010   format %{ "cmp  $op1, $op2" %}
16011 
16012   ins_encode(aarch64_enc_cmp_imm(op1, op2));
16013 
16014   ins_pipe(icmp_reg_imm);
16015 %}
16016 
16017 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
16018 %{
16019   match(Set cr (CmpP op1 op2));
16020 
16021   effect(DEF cr, USE op1, USE op2);
16022 
16023   ins_cost(INSN_COST);
16024   format %{ "cmp  $op1, $op2\t // ptr" %}
16025 
16026   ins_encode(aarch64_enc_cmpp(op1, op2));
16027 
16028   ins_pipe(icmp_reg_reg);
16029 %}
16030 
16031 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
16032 %{
16033   match(Set cr (CmpN op1 op2));
16034 
16035   effect(DEF cr, USE op1, USE op2);
16036 
16037   ins_cost(INSN_COST);
16038   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
16039 
16040   ins_encode(aarch64_enc_cmpn(op1, op2));
16041 
16042   ins_pipe(icmp_reg_reg);
16043 %}
16044 
16045 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
16046 %{
16047   match(Set cr (CmpP op1 zero));
16048 
16049   effect(DEF cr, USE op1, USE zero);
16050 
16051   ins_cost(INSN_COST);
16052   format %{ "cmp  $op1, 0\t // ptr" %}
16053 
16054   ins_encode(aarch64_enc_testp(op1));
16055 
16056   ins_pipe(icmp_reg_imm);
16057 %}
16058 
16059 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
16060 %{
16061   match(Set cr (CmpN op1 zero));
16062 
16063   effect(DEF cr, USE op1, USE zero);
16064 
16065   ins_cost(INSN_COST);
16066   format %{ "cmp  $op1, 0\t // compressed ptr" %}
16067 
16068   ins_encode(aarch64_enc_testn(op1));
16069 
16070   ins_pipe(icmp_reg_imm);
16071 %}
16072 
16073 // FP comparisons
16074 //
16075 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
16076 // using normal cmpOp. See declaration of rFlagsReg for details.
16077 
16078 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
16079 %{
16080   match(Set cr (CmpF src1 src2));
16081 
16082   ins_cost(3 * INSN_COST);
16083   format %{ "fcmps $src1, $src2" %}
16084 
16085   ins_encode %{
16086     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16087   %}
16088 
16089   ins_pipe(pipe_class_compare);
16090 %}
16091 
16092 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
16093 %{
16094   match(Set cr (CmpF src1 src2));
16095 
16096   ins_cost(3 * INSN_COST);
16097   format %{ "fcmps $src1, 0.0" %}
16098 
16099   ins_encode %{
16100     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
16101   %}
16102 
16103   ins_pipe(pipe_class_compare);
16104 %}
16105 // FROM HERE
16106 
16107 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
16108 %{
16109   match(Set cr (CmpD src1 src2));
16110 
16111   ins_cost(3 * INSN_COST);
16112   format %{ "fcmpd $src1, $src2" %}
16113 
16114   ins_encode %{
16115     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16116   %}
16117 
16118   ins_pipe(pipe_class_compare);
16119 %}
16120 
16121 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
16122 %{
16123   match(Set cr (CmpD src1 src2));
16124 
16125   ins_cost(3 * INSN_COST);
16126   format %{ "fcmpd $src1, 0.0" %}
16127 
16128   ins_encode %{
16129     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16130   %}
16131 
16132   ins_pipe(pipe_class_compare);
16133 %}
16134 
16135 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16136 %{
16137   match(Set dst (CmpF3 src1 src2));
16138   effect(KILL cr);
16139 
16140   ins_cost(5 * INSN_COST);
16141   format %{ "fcmps $src1, $src2\n\t"
16142             "csinvw($dst, zr, zr, eq\n\t"
16143             "csnegw($dst, $dst, $dst, lt)"
16144   %}
16145 
16146   ins_encode %{
16147     Label done;
16148     FloatRegister s1 = as_FloatRegister($src1$$reg);
16149     FloatRegister s2 = as_FloatRegister($src2$$reg);
16150     Register d = as_Register($dst$$reg);
16151     __ fcmps(s1, s2);
16152     // installs 0 if EQ else -1
16153     __ csinvw(d, zr, zr, Assembler::EQ);
16154     // keeps -1 if less or unordered else installs 1
16155     __ csnegw(d, d, d, Assembler::LT);
16156     __ bind(done);
16157   %}
16158 
16159   ins_pipe(pipe_class_default);
16160 
16161 %}
16162 
16163 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16164 %{
16165   match(Set dst (CmpD3 src1 src2));
16166   effect(KILL cr);
16167 
16168   ins_cost(5 * INSN_COST);
16169   format %{ "fcmpd $src1, $src2\n\t"
16170             "csinvw($dst, zr, zr, eq\n\t"
16171             "csnegw($dst, $dst, $dst, lt)"
16172   %}
16173 
16174   ins_encode %{
16175     Label done;
16176     FloatRegister s1 = as_FloatRegister($src1$$reg);
16177     FloatRegister s2 = as_FloatRegister($src2$$reg);
16178     Register d = as_Register($dst$$reg);
16179     __ fcmpd(s1, s2);
16180     // installs 0 if EQ else -1
16181     __ csinvw(d, zr, zr, Assembler::EQ);
16182     // keeps -1 if less or unordered else installs 1
16183     __ csnegw(d, d, d, Assembler::LT);
16184     __ bind(done);
16185   %}
16186   ins_pipe(pipe_class_default);
16187 
16188 %}
16189 
16190 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16191 %{
16192   match(Set dst (CmpF3 src1 zero));
16193   effect(KILL cr);
16194 
16195   ins_cost(5 * INSN_COST);
16196   format %{ "fcmps $src1, 0.0\n\t"
16197             "csinvw($dst, zr, zr, eq\n\t"
16198             "csnegw($dst, $dst, $dst, lt)"
16199   %}
16200 
16201   ins_encode %{
16202     Label done;
16203     FloatRegister s1 = as_FloatRegister($src1$$reg);
16204     Register d = as_Register($dst$$reg);
16205     __ fcmps(s1, 0.0);
16206     // installs 0 if EQ else -1
16207     __ csinvw(d, zr, zr, Assembler::EQ);
16208     // keeps -1 if less or unordered else installs 1
16209     __ csnegw(d, d, d, Assembler::LT);
16210     __ bind(done);
16211   %}
16212 
16213   ins_pipe(pipe_class_default);
16214 
16215 %}
16216 
16217 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16218 %{
16219   match(Set dst (CmpD3 src1 zero));
16220   effect(KILL cr);
16221 
16222   ins_cost(5 * INSN_COST);
16223   format %{ "fcmpd $src1, 0.0\n\t"
16224             "csinvw($dst, zr, zr, eq\n\t"
16225             "csnegw($dst, $dst, $dst, lt)"
16226   %}
16227 
16228   ins_encode %{
16229     Label done;
16230     FloatRegister s1 = as_FloatRegister($src1$$reg);
16231     Register d = as_Register($dst$$reg);
16232     __ fcmpd(s1, 0.0);
16233     // installs 0 if EQ else -1
16234     __ csinvw(d, zr, zr, Assembler::EQ);
16235     // keeps -1 if less or unordered else installs 1
16236     __ csnegw(d, d, d, Assembler::LT);
16237     __ bind(done);
16238   %}
16239   ins_pipe(pipe_class_default);
16240 
16241 %}
16242 
16243 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16244 %{
16245   match(Set dst (CmpLTMask p q));
16246   effect(KILL cr);
16247 
16248   ins_cost(3 * INSN_COST);
16249 
16250   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16251             "csetw $dst, lt\n\t"
16252             "subw $dst, zr, $dst"
16253   %}
16254 
16255   ins_encode %{
16256     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16257     __ csetw(as_Register($dst$$reg), Assembler::LT);
16258     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16259   %}
16260 
16261   ins_pipe(ialu_reg_reg);
16262 %}
16263 
16264 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16265 %{
16266   match(Set dst (CmpLTMask src zero));
16267   effect(KILL cr);
16268 
16269   ins_cost(INSN_COST);
16270 
16271   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16272 
16273   ins_encode %{
16274     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16275   %}
16276 
16277   ins_pipe(ialu_reg_shift);
16278 %}
16279 
16280 // ============================================================================
16281 // Max and Min
16282 
16283 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16284 
16285 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16286 %{
16287   effect(DEF cr, USE src);
16288   ins_cost(INSN_COST);
16289   format %{ "cmpw $src, 0" %}
16290 
16291   ins_encode %{
16292     __ cmpw($src$$Register, 0);
16293   %}
16294   ins_pipe(icmp_reg_imm);
16295 %}
16296 
16297 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16298 %{
16299   match(Set dst (MinI src1 src2));
16300   ins_cost(INSN_COST * 3);
16301 
16302   expand %{
16303     rFlagsReg cr;
16304     compI_reg_reg(cr, src1, src2);
16305     cmovI_reg_reg_lt(dst, src1, src2, cr);
16306   %}
16307 %}
16308 
16309 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16310 %{
16311   match(Set dst (MaxI src1 src2));
16312   ins_cost(INSN_COST * 3);
16313 
16314   expand %{
16315     rFlagsReg cr;
16316     compI_reg_reg(cr, src1, src2);
16317     cmovI_reg_reg_gt(dst, src1, src2, cr);
16318   %}
16319 %}
16320 
16321 
16322 // ============================================================================
16323 // Branch Instructions
16324 
16325 // Direct Branch.
16326 instruct branch(label lbl)
16327 %{
16328   match(Goto);
16329 
16330   effect(USE lbl);
16331 
16332   ins_cost(BRANCH_COST);
16333   format %{ "b  $lbl" %}
16334 
16335   ins_encode(aarch64_enc_b(lbl));
16336 
16337   ins_pipe(pipe_branch);
16338 %}
16339 
16340 // Conditional Near Branch
16341 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16342 %{
16343   // Same match rule as `branchConFar'.
16344   match(If cmp cr);
16345 
16346   effect(USE lbl);
16347 
16348   ins_cost(BRANCH_COST);
16349   // If set to 1 this indicates that the current instruction is a
16350   // short variant of a long branch. This avoids using this
16351   // instruction in first-pass matching. It will then only be used in
16352   // the `Shorten_branches' pass.
16353   // ins_short_branch(1);
16354   format %{ "b$cmp  $lbl" %}
16355 
16356   ins_encode(aarch64_enc_br_con(cmp, lbl));
16357 
16358   ins_pipe(pipe_branch_cond);
16359 %}
16360 
16361 // Conditional Near Branch Unsigned
16362 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16363 %{
16364   // Same match rule as `branchConFar'.
16365   match(If cmp cr);
16366 
16367   effect(USE lbl);
16368 
16369   ins_cost(BRANCH_COST);
16370   // If set to 1 this indicates that the current instruction is a
16371   // short variant of a long branch. This avoids using this
16372   // instruction in first-pass matching. It will then only be used in
16373   // the `Shorten_branches' pass.
16374   // ins_short_branch(1);
16375   format %{ "b$cmp  $lbl\t# unsigned" %}
16376 
16377   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16378 
16379   ins_pipe(pipe_branch_cond);
16380 %}
16381 
16382 // Make use of CBZ and CBNZ.  These instructions, as well as being
16383 // shorter than (cmp; branch), have the additional benefit of not
16384 // killing the flags.
16385 
16386 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16387   match(If cmp (CmpI op1 op2));
16388   effect(USE labl);
16389 
16390   ins_cost(BRANCH_COST);
16391   format %{ "cbw$cmp   $op1, $labl" %}
16392   ins_encode %{
16393     Label* L = $labl$$label;
16394     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16395     if (cond == Assembler::EQ)
16396       __ cbzw($op1$$Register, *L);
16397     else
16398       __ cbnzw($op1$$Register, *L);
16399   %}
16400   ins_pipe(pipe_cmp_branch);
16401 %}
16402 
16403 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16404   match(If cmp (CmpL op1 op2));
16405   effect(USE labl);
16406 
16407   ins_cost(BRANCH_COST);
16408   format %{ "cb$cmp   $op1, $labl" %}
16409   ins_encode %{
16410     Label* L = $labl$$label;
16411     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16412     if (cond == Assembler::EQ)
16413       __ cbz($op1$$Register, *L);
16414     else
16415       __ cbnz($op1$$Register, *L);
16416   %}
16417   ins_pipe(pipe_cmp_branch);
16418 %}
16419 
16420 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16421   match(If cmp (CmpP op1 op2));
16422   effect(USE labl);
16423 
16424   ins_cost(BRANCH_COST);
16425   format %{ "cb$cmp   $op1, $labl" %}
16426   ins_encode %{
16427     Label* L = $labl$$label;
16428     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16429     if (cond == Assembler::EQ)
16430       __ cbz($op1$$Register, *L);
16431     else
16432       __ cbnz($op1$$Register, *L);
16433   %}
16434   ins_pipe(pipe_cmp_branch);
16435 %}
16436 
16437 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16438   match(If cmp (CmpN op1 op2));
16439   effect(USE labl);
16440 
16441   ins_cost(BRANCH_COST);
16442   format %{ "cbw$cmp   $op1, $labl" %}
16443   ins_encode %{
16444     Label* L = $labl$$label;
16445     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16446     if (cond == Assembler::EQ)
16447       __ cbzw($op1$$Register, *L);
16448     else
16449       __ cbnzw($op1$$Register, *L);
16450   %}
16451   ins_pipe(pipe_cmp_branch);
16452 %}
16453 
16454 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16455   match(If cmp (CmpP (DecodeN oop) zero));
16456   effect(USE labl);
16457 
16458   ins_cost(BRANCH_COST);
16459   format %{ "cb$cmp   $oop, $labl" %}
16460   ins_encode %{
16461     Label* L = $labl$$label;
16462     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16463     if (cond == Assembler::EQ)
16464       __ cbzw($oop$$Register, *L);
16465     else
16466       __ cbnzw($oop$$Register, *L);
16467   %}
16468   ins_pipe(pipe_cmp_branch);
16469 %}
16470 
16471 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16472   match(If cmp (CmpU op1 op2));
16473   effect(USE labl);
16474 
16475   ins_cost(BRANCH_COST);
16476   format %{ "cbw$cmp   $op1, $labl" %}
16477   ins_encode %{
16478     Label* L = $labl$$label;
16479     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16480     if (cond == Assembler::EQ || cond == Assembler::LS)
16481       __ cbzw($op1$$Register, *L);
16482     else
16483       __ cbnzw($op1$$Register, *L);
16484   %}
16485   ins_pipe(pipe_cmp_branch);
16486 %}
16487 
16488 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16489   match(If cmp (CmpUL op1 op2));
16490   effect(USE labl);
16491 
16492   ins_cost(BRANCH_COST);
16493   format %{ "cb$cmp   $op1, $labl" %}
16494   ins_encode %{
16495     Label* L = $labl$$label;
16496     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16497     if (cond == Assembler::EQ || cond == Assembler::LS)
16498       __ cbz($op1$$Register, *L);
16499     else
16500       __ cbnz($op1$$Register, *L);
16501   %}
16502   ins_pipe(pipe_cmp_branch);
16503 %}
16504 
16505 // Test bit and Branch
16506 
16507 // Patterns for short (< 32KiB) variants
16508 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16509   match(If cmp (CmpL op1 op2));
16510   effect(USE labl);
16511 
16512   ins_cost(BRANCH_COST);
16513   format %{ "cb$cmp   $op1, $labl # long" %}
16514   ins_encode %{
16515     Label* L = $labl$$label;
16516     Assembler::Condition cond =
16517       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16518     __ tbr(cond, $op1$$Register, 63, *L);
16519   %}
16520   ins_pipe(pipe_cmp_branch);
16521   ins_short_branch(1);
16522 %}
16523 
16524 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16525   match(If cmp (CmpI op1 op2));
16526   effect(USE labl);
16527 
16528   ins_cost(BRANCH_COST);
16529   format %{ "cb$cmp   $op1, $labl # int" %}
16530   ins_encode %{
16531     Label* L = $labl$$label;
16532     Assembler::Condition cond =
16533       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16534     __ tbr(cond, $op1$$Register, 31, *L);
16535   %}
16536   ins_pipe(pipe_cmp_branch);
16537   ins_short_branch(1);
16538 %}
16539 
16540 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16541   match(If cmp (CmpL (AndL op1 op2) op3));
16542   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16543   effect(USE labl);
16544 
16545   ins_cost(BRANCH_COST);
16546   format %{ "tb$cmp   $op1, $op2, $labl" %}
16547   ins_encode %{
16548     Label* L = $labl$$label;
16549     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16550     int bit = exact_log2_long($op2$$constant);
16551     __ tbr(cond, $op1$$Register, bit, *L);
16552   %}
16553   ins_pipe(pipe_cmp_branch);
16554   ins_short_branch(1);
16555 %}
16556 
16557 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16558   match(If cmp (CmpI (AndI op1 op2) op3));
16559   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16560   effect(USE labl);
16561 
16562   ins_cost(BRANCH_COST);
16563   format %{ "tb$cmp   $op1, $op2, $labl" %}
16564   ins_encode %{
16565     Label* L = $labl$$label;
16566     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16567     int bit = exact_log2((juint)$op2$$constant);
16568     __ tbr(cond, $op1$$Register, bit, *L);
16569   %}
16570   ins_pipe(pipe_cmp_branch);
16571   ins_short_branch(1);
16572 %}
16573 
16574 // And far variants
16575 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16576   match(If cmp (CmpL op1 op2));
16577   effect(USE labl);
16578 
16579   ins_cost(BRANCH_COST);
16580   format %{ "cb$cmp   $op1, $labl # long" %}
16581   ins_encode %{
16582     Label* L = $labl$$label;
16583     Assembler::Condition cond =
16584       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16585     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16586   %}
16587   ins_pipe(pipe_cmp_branch);
16588 %}
16589 
16590 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16591   match(If cmp (CmpI op1 op2));
16592   effect(USE labl);
16593 
16594   ins_cost(BRANCH_COST);
16595   format %{ "cb$cmp   $op1, $labl # int" %}
16596   ins_encode %{
16597     Label* L = $labl$$label;
16598     Assembler::Condition cond =
16599       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16600     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16601   %}
16602   ins_pipe(pipe_cmp_branch);
16603 %}
16604 
16605 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16606   match(If cmp (CmpL (AndL op1 op2) op3));
16607   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16608   effect(USE labl);
16609 
16610   ins_cost(BRANCH_COST);
16611   format %{ "tb$cmp   $op1, $op2, $labl" %}
16612   ins_encode %{
16613     Label* L = $labl$$label;
16614     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16615     int bit = exact_log2_long($op2$$constant);
16616     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16617   %}
16618   ins_pipe(pipe_cmp_branch);
16619 %}
16620 
16621 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16622   match(If cmp (CmpI (AndI op1 op2) op3));
16623   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16624   effect(USE labl);
16625 
16626   ins_cost(BRANCH_COST);
16627   format %{ "tb$cmp   $op1, $op2, $labl" %}
16628   ins_encode %{
16629     Label* L = $labl$$label;
16630     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16631     int bit = exact_log2((juint)$op2$$constant);
16632     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16633   %}
16634   ins_pipe(pipe_cmp_branch);
16635 %}
16636 
16637 // Test bits
16638 
16639 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16640   match(Set cr (CmpL (AndL op1 op2) op3));
16641   predicate(Assembler::operand_valid_for_logical_immediate
16642             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16643 
16644   ins_cost(INSN_COST);
16645   format %{ "tst $op1, $op2 # long" %}
16646   ins_encode %{
16647     __ tst($op1$$Register, $op2$$constant);
16648   %}
16649   ins_pipe(ialu_reg_reg);
16650 %}
16651 
16652 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16653   match(Set cr (CmpI (AndI op1 op2) op3));
16654   predicate(Assembler::operand_valid_for_logical_immediate
16655             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16656 
16657   ins_cost(INSN_COST);
16658   format %{ "tst $op1, $op2 # int" %}
16659   ins_encode %{
16660     __ tstw($op1$$Register, $op2$$constant);
16661   %}
16662   ins_pipe(ialu_reg_reg);
16663 %}
16664 
16665 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16666   match(Set cr (CmpL (AndL op1 op2) op3));
16667 
16668   ins_cost(INSN_COST);
16669   format %{ "tst $op1, $op2 # long" %}
16670   ins_encode %{
16671     __ tst($op1$$Register, $op2$$Register);
16672   %}
16673   ins_pipe(ialu_reg_reg);
16674 %}
16675 
16676 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16677   match(Set cr (CmpI (AndI op1 op2) op3));
16678 
16679   ins_cost(INSN_COST);
16680   format %{ "tstw $op1, $op2 # int" %}
16681   ins_encode %{
16682     __ tstw($op1$$Register, $op2$$Register);
16683   %}
16684   ins_pipe(ialu_reg_reg);
16685 %}
16686 
16687 
16688 // Conditional Far Branch
16689 // Conditional Far Branch Unsigned
16690 // TODO: fixme
16691 
16692 // counted loop end branch near
16693 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16694 %{
16695   match(CountedLoopEnd cmp cr);
16696 
16697   effect(USE lbl);
16698 
16699   ins_cost(BRANCH_COST);
16700   // short variant.
16701   // ins_short_branch(1);
16702   format %{ "b$cmp $lbl \t// counted loop end" %}
16703 
16704   ins_encode(aarch64_enc_br_con(cmp, lbl));
16705 
16706   ins_pipe(pipe_branch);
16707 %}
16708 
16709 // counted loop end branch near Unsigned
16710 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16711 %{
16712   match(CountedLoopEnd cmp cr);
16713 
16714   effect(USE lbl);
16715 
16716   ins_cost(BRANCH_COST);
16717   // short variant.
16718   // ins_short_branch(1);
16719   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16720 
16721   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16722 
16723   ins_pipe(pipe_branch);
16724 %}
16725 
16726 // counted loop end branch far
16727 // counted loop end branch far unsigned
16728 // TODO: fixme
16729 
16730 // ============================================================================
16731 // inlined locking and unlocking
16732 
16733 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16734 %{
16735   match(Set cr (FastLock object box));
16736   effect(TEMP tmp, TEMP tmp2);
16737 
16738   // TODO
16739   // identify correct cost
16740   ins_cost(5 * INSN_COST);
16741   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16742 
16743   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16744 
16745   ins_pipe(pipe_serial);
16746 %}
16747 
16748 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16749 %{
16750   match(Set cr (FastUnlock object box));
16751   effect(TEMP tmp, TEMP tmp2);
16752 
16753   ins_cost(5 * INSN_COST);
16754   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16755 
16756   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16757 
16758   ins_pipe(pipe_serial);
16759 %}
16760 
16761 
16762 // ============================================================================
16763 // Safepoint Instructions
16764 
16765 // TODO
16766 // provide a near and far version of this code
16767 
16768 instruct safePoint(rFlagsReg cr, iRegP poll)
16769 %{
16770   match(SafePoint poll);
16771   effect(KILL cr);
16772 
16773   format %{
16774     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16775   %}
16776   ins_encode %{
16777     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16778   %}
16779   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16780 %}
16781 
16782 
16783 // ============================================================================
16784 // Procedure Call/Return Instructions
16785 
16786 // Call Java Static Instruction
16787 
16788 instruct CallStaticJavaDirect(method meth)
16789 %{
16790   match(CallStaticJava);
16791 
16792   effect(USE meth);
16793 
16794   ins_cost(CALL_COST);
16795 
16796   format %{ "call,static $meth \t// ==> " %}
16797 
16798   ins_encode(aarch64_enc_java_static_call(meth),
16799              aarch64_enc_call_epilog);
16800 
16801   ins_pipe(pipe_class_call);
16802 %}
16803 
16804 // TO HERE
16805 
16806 // Call Java Dynamic Instruction
16807 instruct CallDynamicJavaDirect(method meth)
16808 %{
16809   match(CallDynamicJava);
16810 
16811   effect(USE meth);
16812 
16813   ins_cost(CALL_COST);
16814 
16815   format %{ "CALL,dynamic $meth \t// ==> " %}
16816 
16817   ins_encode(aarch64_enc_java_dynamic_call(meth),
16818              aarch64_enc_call_epilog);
16819 
16820   ins_pipe(pipe_class_call);
16821 %}
16822 
16823 // Call Runtime Instruction
16824 
16825 instruct CallRuntimeDirect(method meth)
16826 %{
16827   match(CallRuntime);
16828 
16829   effect(USE meth);
16830 
16831   ins_cost(CALL_COST);
16832 
16833   format %{ "CALL, runtime $meth" %}
16834 
16835   ins_encode( aarch64_enc_java_to_runtime(meth) );
16836 
16837   ins_pipe(pipe_class_call);
16838 %}
16839 
16840 // Call Runtime Instruction
16841 
16842 instruct CallLeafDirect(method meth)
16843 %{
16844   match(CallLeaf);
16845 
16846   effect(USE meth);
16847 
16848   ins_cost(CALL_COST);
16849 
16850   format %{ "CALL, runtime leaf $meth" %}
16851 
16852   ins_encode( aarch64_enc_java_to_runtime(meth) );
16853 
16854   ins_pipe(pipe_class_call);
16855 %}
16856 
16857 // Call Runtime Instruction
16858 
16859 instruct CallLeafNoFPDirect(method meth)
16860 %{
16861   match(CallLeafNoFP);
16862 
16863   effect(USE meth);
16864 
16865   ins_cost(CALL_COST);
16866 
16867   format %{ "CALL, runtime leaf nofp $meth" %}
16868 
16869   ins_encode( aarch64_enc_java_to_runtime(meth) );
16870 
16871   ins_pipe(pipe_class_call);
16872 %}
16873 
16874 instruct CallNativeDirect(method meth)
16875 %{
16876   match(CallNative);
16877 
16878   effect(USE meth);
16879 
16880   ins_cost(CALL_COST);
16881 
16882   format %{ "CALL, native $meth" %}
16883 
16884   ins_encode( aarch64_enc_java_to_runtime(meth) );
16885 
16886   ins_pipe(pipe_class_call);
16887 %}
16888 
16889 // Tail Call; Jump from runtime stub to Java code.
16890 // Also known as an 'interprocedural jump'.
16891 // Target of jump will eventually return to caller.
16892 // TailJump below removes the return address.
16893 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16894 %{
16895   match(TailCall jump_target method_ptr);
16896 
16897   ins_cost(CALL_COST);
16898 
16899   format %{ "br $jump_target\t# $method_ptr holds method" %}
16900 
16901   ins_encode(aarch64_enc_tail_call(jump_target));
16902 
16903   ins_pipe(pipe_class_call);
16904 %}
16905 
16906 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16907 %{
16908   match(TailJump jump_target ex_oop);
16909 
16910   ins_cost(CALL_COST);
16911 
16912   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16913 
16914   ins_encode(aarch64_enc_tail_jmp(jump_target));
16915 
16916   ins_pipe(pipe_class_call);
16917 %}
16918 
16919 // Create exception oop: created by stack-crawling runtime code.
16920 // Created exception is now available to this handler, and is setup
16921 // just prior to jumping to this handler. No code emitted.
16922 // TODO check
16923 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16924 instruct CreateException(iRegP_R0 ex_oop)
16925 %{
16926   match(Set ex_oop (CreateEx));
16927 
16928   format %{ " -- \t// exception oop; no code emitted" %}
16929 
16930   size(0);
16931 
16932   ins_encode( /*empty*/ );
16933 
16934   ins_pipe(pipe_class_empty);
16935 %}
16936 
16937 // Rethrow exception: The exception oop will come in the first
16938 // argument position. Then JUMP (not call) to the rethrow stub code.
16939 instruct RethrowException() %{
16940   match(Rethrow);
16941   ins_cost(CALL_COST);
16942 
16943   format %{ "b rethrow_stub" %}
16944 
16945   ins_encode( aarch64_enc_rethrow() );
16946 
16947   ins_pipe(pipe_class_call);
16948 %}
16949 
16950 
16951 // Return Instruction
16952 // epilog node loads ret address into lr as part of frame pop
16953 instruct Ret()
16954 %{
16955   match(Return);
16956 
16957   format %{ "ret\t// return register" %}
16958 
16959   ins_encode( aarch64_enc_ret() );
16960 
16961   ins_pipe(pipe_branch);
16962 %}
16963 
16964 // Die now.
16965 instruct ShouldNotReachHere() %{
16966   match(Halt);
16967 
16968   ins_cost(CALL_COST);
16969   format %{ "ShouldNotReachHere" %}
16970 
16971   ins_encode %{
16972     if (is_reachable()) {
16973       __ stop(_halt_reason);
16974     }
16975   %}
16976 
16977   ins_pipe(pipe_class_default);
16978 %}
16979 
16980 // ============================================================================
16981 // Partial Subtype Check
16982 //
16983 // superklass array for an instance of the superklass.  Set a hidden
16984 // internal cache on a hit (cache is checked with exposed code in
16985 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16986 // encoding ALSO sets flags.
16987 
16988 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16989 %{
16990   match(Set result (PartialSubtypeCheck sub super));
16991   effect(KILL cr, KILL temp);
16992 
16993   ins_cost(1100);  // slightly larger than the next version
16994   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16995 
16996   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16997 
16998   opcode(0x1); // Force zero of result reg on hit
16999 
17000   ins_pipe(pipe_class_memory);
17001 %}
17002 
17003 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
17004 %{
17005   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
17006   effect(KILL temp, KILL result);
17007 
17008   ins_cost(1100);  // slightly larger than the next version
17009   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
17010 
17011   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
17012 
17013   opcode(0x0); // Don't zero result reg on hit
17014 
17015   ins_pipe(pipe_class_memory);
17016 %}
17017 
17018 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17019                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
17020 %{
17021   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
17022   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17023   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17024 
17025   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
17026   ins_encode %{
17027     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17028     __ string_compare($str1$$Register, $str2$$Register,
17029                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17030                       $tmp1$$Register, $tmp2$$Register,
17031                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
17032   %}
17033   ins_pipe(pipe_class_memory);
17034 %}
17035 
17036 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17037                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
17038 %{
17039   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
17040   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17041   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17042 
17043   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
17044   ins_encode %{
17045     __ string_compare($str1$$Register, $str2$$Register,
17046                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17047                       $tmp1$$Register, $tmp2$$Register,
17048                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
17049   %}
17050   ins_pipe(pipe_class_memory);
17051 %}
17052 
17053 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17054                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17055                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
17056 %{
17057   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
17058   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17059   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
17060          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17061 
17062   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
17063   ins_encode %{
17064     __ string_compare($str1$$Register, $str2$$Register,
17065                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17066                       $tmp1$$Register, $tmp2$$Register,
17067                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
17068                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
17069   %}
17070   ins_pipe(pipe_class_memory);
17071 %}
17072 
17073 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17074                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17075                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
17076 %{
17077   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
17078   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17079   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
17080          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17081 
17082   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
17083   ins_encode %{
17084     __ string_compare($str1$$Register, $str2$$Register,
17085                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17086                       $tmp1$$Register, $tmp2$$Register,
17087                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
17088                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
17089   %}
17090   ins_pipe(pipe_class_memory);
17091 %}
17092 
17093 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17094                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17095                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17096                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17097 %{
17098   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17099   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17100   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17101          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17102          TEMP vtmp0, TEMP vtmp1, KILL cr);
17103   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
17104             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17105 
17106   ins_encode %{
17107     __ string_indexof($str1$$Register, $str2$$Register,
17108                       $cnt1$$Register, $cnt2$$Register,
17109                       $tmp1$$Register, $tmp2$$Register,
17110                       $tmp3$$Register, $tmp4$$Register,
17111                       $tmp5$$Register, $tmp6$$Register,
17112                       -1, $result$$Register, StrIntrinsicNode::UU);
17113   %}
17114   ins_pipe(pipe_class_memory);
17115 %}
17116 
17117 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17118                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17119                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17120                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17121 %{
17122   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17123   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17124   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17125          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17126          TEMP vtmp0, TEMP vtmp1, KILL cr);
17127   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
17128             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17129 
17130   ins_encode %{
17131     __ string_indexof($str1$$Register, $str2$$Register,
17132                       $cnt1$$Register, $cnt2$$Register,
17133                       $tmp1$$Register, $tmp2$$Register,
17134                       $tmp3$$Register, $tmp4$$Register,
17135                       $tmp5$$Register, $tmp6$$Register,
17136                       -1, $result$$Register, StrIntrinsicNode::LL);
17137   %}
17138   ins_pipe(pipe_class_memory);
17139 %}
17140 
17141 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17142                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
17143                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17144                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17145 %{
17146   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17147   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17148   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17149          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
17150          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
17151   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
17152             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17153 
17154   ins_encode %{
17155     __ string_indexof($str1$$Register, $str2$$Register,
17156                       $cnt1$$Register, $cnt2$$Register,
17157                       $tmp1$$Register, $tmp2$$Register,
17158                       $tmp3$$Register, $tmp4$$Register,
17159                       $tmp5$$Register, $tmp6$$Register,
17160                       -1, $result$$Register, StrIntrinsicNode::UL);
17161   %}
17162   ins_pipe(pipe_class_memory);
17163 %}
17164 
17165 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17166                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17167                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17168 %{
17169   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17170   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17171   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17172          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17173   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
17174             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17175 
17176   ins_encode %{
17177     int icnt2 = (int)$int_cnt2$$constant;
17178     __ string_indexof($str1$$Register, $str2$$Register,
17179                       $cnt1$$Register, zr,
17180                       $tmp1$$Register, $tmp2$$Register,
17181                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17182                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17183   %}
17184   ins_pipe(pipe_class_memory);
17185 %}
17186 
17187 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17188                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17189                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17190 %{
17191   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17192   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17193   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17194          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17195   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17196             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17197 
17198   ins_encode %{
17199     int icnt2 = (int)$int_cnt2$$constant;
17200     __ string_indexof($str1$$Register, $str2$$Register,
17201                       $cnt1$$Register, zr,
17202                       $tmp1$$Register, $tmp2$$Register,
17203                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17204                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17205   %}
17206   ins_pipe(pipe_class_memory);
17207 %}
17208 
17209 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17210                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17211                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17212 %{
17213   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17214   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17215   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17216          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17217   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17218             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17219 
17220   ins_encode %{
17221     int icnt2 = (int)$int_cnt2$$constant;
17222     __ string_indexof($str1$$Register, $str2$$Register,
17223                       $cnt1$$Register, zr,
17224                       $tmp1$$Register, $tmp2$$Register,
17225                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17226                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17227   %}
17228   ins_pipe(pipe_class_memory);
17229 %}
17230 
17231 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17232                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17233                               iRegINoSp tmp3, rFlagsReg cr)
17234 %{
17235   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17236   predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17237   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17238          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17239 
17240   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17241 
17242   ins_encode %{
17243     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17244                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17245                            $tmp3$$Register);
17246   %}
17247   ins_pipe(pipe_class_memory);
17248 %}
17249 
17250 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17251                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17252                               iRegINoSp tmp3, rFlagsReg cr)
17253 %{
17254   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17255   predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17256   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17257          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17258 
17259   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17260 
17261   ins_encode %{
17262     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17263                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17264                            $tmp3$$Register);
17265   %}
17266   ins_pipe(pipe_class_memory);
17267 %}
17268 
17269 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17270                         iRegI_R0 result, rFlagsReg cr)
17271 %{
17272   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17273   match(Set result (StrEquals (Binary str1 str2) cnt));
17274   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17275 
17276   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17277   ins_encode %{
17278     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17279     __ string_equals($str1$$Register, $str2$$Register,
17280                      $result$$Register, $cnt$$Register, 1);
17281   %}
17282   ins_pipe(pipe_class_memory);
17283 %}
17284 
17285 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17286                         iRegI_R0 result, rFlagsReg cr)
17287 %{
17288   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17289   match(Set result (StrEquals (Binary str1 str2) cnt));
17290   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17291 
17292   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17293   ins_encode %{
17294     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17295     __ string_equals($str1$$Register, $str2$$Register,
17296                      $result$$Register, $cnt$$Register, 2);
17297   %}
17298   ins_pipe(pipe_class_memory);
17299 %}
17300 
17301 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17302                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17303                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17304                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17305                        iRegP_R10 tmp, rFlagsReg cr)
17306 %{
17307   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17308   match(Set result (AryEq ary1 ary2));
17309   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17310          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17311          TEMP vtmp6, TEMP vtmp7, KILL cr);
17312 
17313   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17314   ins_encode %{
17315     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17316                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17317                                    $result$$Register, $tmp$$Register, 1);
17318     if (tpc == NULL) {
17319       ciEnv::current()->record_failure("CodeCache is full");
17320       return;
17321     }
17322   %}
17323   ins_pipe(pipe_class_memory);
17324 %}
17325 
17326 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17327                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17328                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17329                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17330                        iRegP_R10 tmp, rFlagsReg cr)
17331 %{
17332   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17333   match(Set result (AryEq ary1 ary2));
17334   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17335          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17336          TEMP vtmp6, TEMP vtmp7, KILL cr);
17337 
17338   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17339   ins_encode %{
17340     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17341                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17342                                    $result$$Register, $tmp$$Register, 2);
17343     if (tpc == NULL) {
17344       ciEnv::current()->record_failure("CodeCache is full");
17345       return;
17346     }
17347   %}
17348   ins_pipe(pipe_class_memory);
17349 %}
17350 
17351 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17352 %{
17353   match(Set result (HasNegatives ary1 len));
17354   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17355   format %{ "has negatives byte[] $ary1,$len -> $result" %}
17356   ins_encode %{
17357     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
17358     if (tpc == NULL) {
17359       ciEnv::current()->record_failure("CodeCache is full");
17360       return;
17361     }
17362   %}
17363   ins_pipe( pipe_slow );
17364 %}
17365 
17366 // fast char[] to byte[] compression
17367 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17368                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17369                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17370                          iRegI_R0 result, rFlagsReg cr)
17371 %{
17372   match(Set result (StrCompressedCopy src (Binary dst len)));
17373   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17374          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17375 
17376   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17377   ins_encode %{
17378     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17379                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17380                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17381                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17382   %}
17383   ins_pipe(pipe_slow);
17384 %}
17385 
17386 // fast byte[] to char[] inflation
17387 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17388                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17389                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17390 %{
17391   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17392   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17393          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17394          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17395 
17396   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17397   ins_encode %{
17398     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17399                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17400                                         $vtmp2$$FloatRegister, $tmp$$Register);
17401     if (tpc == NULL) {
17402       ciEnv::current()->record_failure("CodeCache is full");
17403       return;
17404     }
17405   %}
17406   ins_pipe(pipe_class_memory);
17407 %}
17408 
17409 // encode char[] to byte[] in ISO_8859_1
17410 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17411                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17412                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17413                           iRegI_R0 result, rFlagsReg cr)
17414 %{
17415   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17416   match(Set result (EncodeISOArray src (Binary dst len)));
17417   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17418          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17419 
17420   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17421   ins_encode %{
17422     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17423                         $result$$Register, false,
17424                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17425                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17426                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17427   %}
17428   ins_pipe(pipe_class_memory);
17429 %}
17430 
17431 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17432                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17433                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17434                             iRegI_R0 result, rFlagsReg cr)
17435 %{
17436   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17437   match(Set result (EncodeISOArray src (Binary dst len)));
17438   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17439          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17440 
17441   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17442   ins_encode %{
17443     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17444                         $result$$Register, true,
17445                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17446                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17447                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17448   %}
17449   ins_pipe(pipe_class_memory);
17450 %}
17451 
17452 // ============================================================================
17453 // This name is KNOWN by the ADLC and cannot be changed.
17454 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17455 // for this guy.
17456 instruct tlsLoadP(thread_RegP dst)
17457 %{
17458   match(Set dst (ThreadLocal));
17459 
17460   ins_cost(0);
17461 
17462   format %{ " -- \t// $dst=Thread::current(), empty" %}
17463 
17464   size(0);
17465 
17466   ins_encode( /*empty*/ );
17467 
17468   ins_pipe(pipe_class_empty);
17469 %}
17470 
17471 //----------PEEPHOLE RULES-----------------------------------------------------
17472 // These must follow all instruction definitions as they use the names
17473 // defined in the instructions definitions.
17474 //
17475 // peepmatch ( root_instr_name [preceding_instruction]* );
17476 //
17477 // peepconstraint %{
17478 // (instruction_number.operand_name relational_op instruction_number.operand_name
17479 //  [, ...] );
17480 // // instruction numbers are zero-based using left to right order in peepmatch
17481 //
17482 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17483 // // provide an instruction_number.operand_name for each operand that appears
17484 // // in the replacement instruction's match rule
17485 //
17486 // ---------VM FLAGS---------------------------------------------------------
17487 //
17488 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17489 //
17490 // Each peephole rule is given an identifying number starting with zero and
17491 // increasing by one in the order seen by the parser.  An individual peephole
17492 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17493 // on the command-line.
17494 //
17495 // ---------CURRENT LIMITATIONS----------------------------------------------
17496 //
17497 // Only match adjacent instructions in same basic block
17498 // Only equality constraints
17499 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17500 // Only one replacement instruction
17501 //
17502 // ---------EXAMPLE----------------------------------------------------------
17503 //
17504 // // pertinent parts of existing instructions in architecture description
17505 // instruct movI(iRegINoSp dst, iRegI src)
17506 // %{
17507 //   match(Set dst (CopyI src));
17508 // %}
17509 //
17510 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17511 // %{
17512 //   match(Set dst (AddI dst src));
17513 //   effect(KILL cr);
17514 // %}
17515 //
17516 // // Change (inc mov) to lea
17517 // peephole %{
17518 //   // increment preceeded by register-register move
17519 //   peepmatch ( incI_iReg movI );
17520 //   // require that the destination register of the increment
17521 //   // match the destination register of the move
17522 //   peepconstraint ( 0.dst == 1.dst );
17523 //   // construct a replacement instruction that sets
17524 //   // the destination to ( move's source register + one )
17525 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17526 // %}
17527 //
17528 
17529 // Implementation no longer uses movX instructions since
17530 // machine-independent system no longer uses CopyX nodes.
17531 //
17532 // peephole
17533 // %{
17534 //   peepmatch (incI_iReg movI);
17535 //   peepconstraint (0.dst == 1.dst);
17536 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17537 // %}
17538 
17539 // peephole
17540 // %{
17541 //   peepmatch (decI_iReg movI);
17542 //   peepconstraint (0.dst == 1.dst);
17543 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17544 // %}
17545 
17546 // peephole
17547 // %{
17548 //   peepmatch (addI_iReg_imm movI);
17549 //   peepconstraint (0.dst == 1.dst);
17550 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17551 // %}
17552 
17553 // peephole
17554 // %{
17555 //   peepmatch (incL_iReg movL);
17556 //   peepconstraint (0.dst == 1.dst);
17557 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17558 // %}
17559 
17560 // peephole
17561 // %{
17562 //   peepmatch (decL_iReg movL);
17563 //   peepconstraint (0.dst == 1.dst);
17564 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17565 // %}
17566 
17567 // peephole
17568 // %{
17569 //   peepmatch (addL_iReg_imm movL);
17570 //   peepconstraint (0.dst == 1.dst);
17571 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17572 // %}
17573 
17574 // peephole
17575 // %{
17576 //   peepmatch (addP_iReg_imm movP);
17577 //   peepconstraint (0.dst == 1.dst);
17578 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17579 // %}
17580 
17581 // // Change load of spilled value to only a spill
17582 // instruct storeI(memory mem, iRegI src)
17583 // %{
17584 //   match(Set mem (StoreI mem src));
17585 // %}
17586 //
17587 // instruct loadI(iRegINoSp dst, memory mem)
17588 // %{
17589 //   match(Set dst (LoadI mem));
17590 // %}
17591 //
17592 
17593 //----------SMARTSPILL RULES---------------------------------------------------
17594 // These must follow all instruction definitions as they use the names
17595 // defined in the instructions definitions.
17596 
17597 // Local Variables:
17598 // mode: c++
17599 // End: