1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return 4 * NativeInstruction::instruction_size;
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298   bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // predicate controlling addressing modes
 1315   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1316 %}
 1317 
 1318 source %{
 1319 
 1320   // Derived RegMask with conditionally allocatable registers
 1321 
 1322   void PhaseOutput::pd_perform_mach_node_analysis() {
 1323   }
 1324 
 1325   int MachNode::pd_alignment_required() const {
 1326     return 1;
 1327   }
 1328 
 1329   int MachNode::compute_padding(int current_offset) const {
 1330     return 0;
 1331   }
 1332 
 1333   RegMask _ANY_REG32_mask;
 1334   RegMask _ANY_REG_mask;
 1335   RegMask _PTR_REG_mask;
 1336   RegMask _NO_SPECIAL_REG32_mask;
 1337   RegMask _NO_SPECIAL_REG_mask;
 1338   RegMask _NO_SPECIAL_PTR_REG_mask;
 1339 
 1340   void reg_mask_init() {
 1341     // We derive below RegMask(s) from the ones which are auto-generated from
 1342     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1343     // registers conditionally reserved.
 1344 
 1345     _ANY_REG32_mask = _ALL_REG32_mask;
 1346     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1347 
 1348     _ANY_REG_mask = _ALL_REG_mask;
 1349 
 1350     _PTR_REG_mask = _ALL_REG_mask;
 1351 
 1352     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1353     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1354 
 1355     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1356     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1357 
 1358     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     // r27 is not allocatable when compressed oops is on and heapbase is not
 1362     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1363     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1364       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1365       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1366       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1367     }
 1368 
 1369     // r29 is not allocatable when PreserveFramePointer is on
 1370     if (PreserveFramePointer) {
 1371       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1372       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1373       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1374     }
 1375   }
 1376 
 1377   // Optimizaton of volatile gets and puts
 1378   // -------------------------------------
 1379   //
 1380   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1381   // use to implement volatile reads and writes. For a volatile read
 1382   // we simply need
 1383   //
 1384   //   ldar<x>
 1385   //
 1386   // and for a volatile write we need
 1387   //
 1388   //   stlr<x>
 1389   //
 1390   // Alternatively, we can implement them by pairing a normal
 1391   // load/store with a memory barrier. For a volatile read we need
 1392   //
 1393   //   ldr<x>
 1394   //   dmb ishld
 1395   //
 1396   // for a volatile write
 1397   //
 1398   //   dmb ish
 1399   //   str<x>
 1400   //   dmb ish
 1401   //
 1402   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1403   // sequences. These are normally translated to an instruction
 1404   // sequence like the following
 1405   //
 1406   //   dmb      ish
 1407   // retry:
 1408   //   ldxr<x>   rval raddr
 1409   //   cmp       rval rold
 1410   //   b.ne done
 1411   //   stlxr<x>  rval, rnew, rold
 1412   //   cbnz      rval retry
 1413   // done:
 1414   //   cset      r0, eq
 1415   //   dmb ishld
 1416   //
 1417   // Note that the exclusive store is already using an stlxr
 1418   // instruction. That is required to ensure visibility to other
 1419   // threads of the exclusive write (assuming it succeeds) before that
 1420   // of any subsequent writes.
 1421   //
 1422   // The following instruction sequence is an improvement on the above
 1423   //
 1424   // retry:
 1425   //   ldaxr<x>  rval raddr
 1426   //   cmp       rval rold
 1427   //   b.ne done
 1428   //   stlxr<x>  rval, rnew, rold
 1429   //   cbnz      rval retry
 1430   // done:
 1431   //   cset      r0, eq
 1432   //
 1433   // We don't need the leading dmb ish since the stlxr guarantees
 1434   // visibility of prior writes in the case that the swap is
 1435   // successful. Crucially we don't have to worry about the case where
 1436   // the swap is not successful since no valid program should be
 1437   // relying on visibility of prior changes by the attempting thread
 1438   // in the case where the CAS fails.
 1439   //
 1440   // Similarly, we don't need the trailing dmb ishld if we substitute
 1441   // an ldaxr instruction since that will provide all the guarantees we
 1442   // require regarding observation of changes made by other threads
 1443   // before any change to the CAS address observed by the load.
 1444   //
 1445   // In order to generate the desired instruction sequence we need to
 1446   // be able to identify specific 'signature' ideal graph node
 1447   // sequences which i) occur as a translation of a volatile reads or
 1448   // writes or CAS operations and ii) do not occur through any other
 1449   // translation or graph transformation. We can then provide
 1450   // alternative aldc matching rules which translate these node
 1451   // sequences to the desired machine code sequences. Selection of the
 1452   // alternative rules can be implemented by predicates which identify
 1453   // the relevant node sequences.
 1454   //
 1455   // The ideal graph generator translates a volatile read to the node
 1456   // sequence
 1457   //
 1458   //   LoadX[mo_acquire]
 1459   //   MemBarAcquire
 1460   //
 1461   // As a special case when using the compressed oops optimization we
 1462   // may also see this variant
 1463   //
 1464   //   LoadN[mo_acquire]
 1465   //   DecodeN
 1466   //   MemBarAcquire
 1467   //
 1468   // A volatile write is translated to the node sequence
 1469   //
 1470   //   MemBarRelease
 1471   //   StoreX[mo_release] {CardMark}-optional
 1472   //   MemBarVolatile
 1473   //
 1474   // n.b. the above node patterns are generated with a strict
 1475   // 'signature' configuration of input and output dependencies (see
 1476   // the predicates below for exact details). The card mark may be as
 1477   // simple as a few extra nodes or, in a few GC configurations, may
 1478   // include more complex control flow between the leading and
 1479   // trailing memory barriers. However, whatever the card mark
 1480   // configuration these signatures are unique to translated volatile
 1481   // reads/stores -- they will not appear as a result of any other
 1482   // bytecode translation or inlining nor as a consequence of
 1483   // optimizing transforms.
 1484   //
 1485   // We also want to catch inlined unsafe volatile gets and puts and
 1486   // be able to implement them using either ldar<x>/stlr<x> or some
 1487   // combination of ldr<x>/stlr<x> and dmb instructions.
 1488   //
 1489   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1490   // normal volatile put node sequence containing an extra cpuorder
 1491   // membar
 1492   //
 1493   //   MemBarRelease
 1494   //   MemBarCPUOrder
 1495   //   StoreX[mo_release] {CardMark}-optional
 1496   //   MemBarCPUOrder
 1497   //   MemBarVolatile
 1498   //
 1499   // n.b. as an aside, a cpuorder membar is not itself subject to
 1500   // matching and translation by adlc rules.  However, the rule
 1501   // predicates need to detect its presence in order to correctly
 1502   // select the desired adlc rules.
 1503   //
 1504   // Inlined unsafe volatile gets manifest as a slightly different
 1505   // node sequence to a normal volatile get because of the
 1506   // introduction of some CPUOrder memory barriers to bracket the
 1507   // Load. However, but the same basic skeleton of a LoadX feeding a
 1508   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1509   // present
 1510   //
 1511   //   MemBarCPUOrder
 1512   //        ||       \\
 1513   //   MemBarCPUOrder LoadX[mo_acquire]
 1514   //        ||            |
 1515   //        ||       {DecodeN} optional
 1516   //        ||       /
 1517   //     MemBarAcquire
 1518   //
 1519   // In this case the acquire membar does not directly depend on the
 1520   // load. However, we can be sure that the load is generated from an
 1521   // inlined unsafe volatile get if we see it dependent on this unique
 1522   // sequence of membar nodes. Similarly, given an acquire membar we
 1523   // can know that it was added because of an inlined unsafe volatile
 1524   // get if it is fed and feeds a cpuorder membar and if its feed
 1525   // membar also feeds an acquiring load.
 1526   //
 1527   // Finally an inlined (Unsafe) CAS operation is translated to the
 1528   // following ideal graph
 1529   //
 1530   //   MemBarRelease
 1531   //   MemBarCPUOrder
 1532   //   CompareAndSwapX {CardMark}-optional
 1533   //   MemBarCPUOrder
 1534   //   MemBarAcquire
 1535   //
 1536   // So, where we can identify these volatile read and write
 1537   // signatures we can choose to plant either of the above two code
 1538   // sequences. For a volatile read we can simply plant a normal
 1539   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1540   // also choose to inhibit translation of the MemBarAcquire and
 1541   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1542   //
 1543   // When we recognise a volatile store signature we can choose to
 1544   // plant at a dmb ish as a translation for the MemBarRelease, a
 1545   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1546   // Alternatively, we can inhibit translation of the MemBarRelease
 1547   // and MemBarVolatile and instead plant a simple stlr<x>
 1548   // instruction.
 1549   //
 1550   // when we recognise a CAS signature we can choose to plant a dmb
 1551   // ish as a translation for the MemBarRelease, the conventional
 1552   // macro-instruction sequence for the CompareAndSwap node (which
 1553   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1554   // Alternatively, we can elide generation of the dmb instructions
 1555   // and plant the alternative CompareAndSwap macro-instruction
 1556   // sequence (which uses ldaxr<x>).
 1557   //
 1558   // Of course, the above only applies when we see these signature
 1559   // configurations. We still want to plant dmb instructions in any
 1560   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1561   // MemBarVolatile. For example, at the end of a constructor which
 1562   // writes final/volatile fields we will see a MemBarRelease
 1563   // instruction and this needs a 'dmb ish' lest we risk the
 1564   // constructed object being visible without making the
 1565   // final/volatile field writes visible.
 1566   //
 1567   // n.b. the translation rules below which rely on detection of the
 1568   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1569   // If we see anything other than the signature configurations we
 1570   // always just translate the loads and stores to ldr<x> and str<x>
 1571   // and translate acquire, release and volatile membars to the
 1572   // relevant dmb instructions.
 1573   //
 1574 
 1575   // is_CAS(int opcode, bool maybe_volatile)
 1576   //
 1577   // return true if opcode is one of the possible CompareAndSwapX
 1578   // values otherwise false.
 1579 
 1580   bool is_CAS(int opcode, bool maybe_volatile)
 1581   {
 1582     switch(opcode) {
 1583       // We handle these
 1584     case Op_CompareAndSwapI:
 1585     case Op_CompareAndSwapL:
 1586     case Op_CompareAndSwapP:
 1587     case Op_CompareAndSwapN:
 1588     case Op_ShenandoahCompareAndSwapP:
 1589     case Op_ShenandoahCompareAndSwapN:
 1590     case Op_CompareAndSwapB:
 1591     case Op_CompareAndSwapS:
 1592     case Op_GetAndSetI:
 1593     case Op_GetAndSetL:
 1594     case Op_GetAndSetP:
 1595     case Op_GetAndSetN:
 1596     case Op_GetAndAddI:
 1597     case Op_GetAndAddL:
 1598       return true;
 1599     case Op_CompareAndExchangeI:
 1600     case Op_CompareAndExchangeN:
 1601     case Op_CompareAndExchangeB:
 1602     case Op_CompareAndExchangeS:
 1603     case Op_CompareAndExchangeL:
 1604     case Op_CompareAndExchangeP:
 1605     case Op_WeakCompareAndSwapB:
 1606     case Op_WeakCompareAndSwapS:
 1607     case Op_WeakCompareAndSwapI:
 1608     case Op_WeakCompareAndSwapL:
 1609     case Op_WeakCompareAndSwapP:
 1610     case Op_WeakCompareAndSwapN:
 1611     case Op_ShenandoahWeakCompareAndSwapP:
 1612     case Op_ShenandoahWeakCompareAndSwapN:
 1613     case Op_ShenandoahCompareAndExchangeP:
 1614     case Op_ShenandoahCompareAndExchangeN:
 1615       return maybe_volatile;
 1616     default:
 1617       return false;
 1618     }
 1619   }
 1620 
 1621   // helper to determine the maximum number of Phi nodes we may need to
 1622   // traverse when searching from a card mark membar for the merge mem
 1623   // feeding a trailing membar or vice versa
 1624 
 1625 // predicates controlling emit of ldr<x>/ldar<x>
 1626 
 1627 bool unnecessary_acquire(const Node *barrier)
 1628 {
 1629   assert(barrier->is_MemBar(), "expecting a membar");
 1630 
 1631   MemBarNode* mb = barrier->as_MemBar();
 1632 
 1633   if (mb->trailing_load()) {
 1634     return true;
 1635   }
 1636 
 1637   if (mb->trailing_load_store()) {
 1638     Node* load_store = mb->in(MemBarNode::Precedent);
 1639     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1640     return is_CAS(load_store->Opcode(), true);
 1641   }
 1642 
 1643   return false;
 1644 }
 1645 
 1646 bool needs_acquiring_load(const Node *n)
 1647 {
 1648   assert(n->is_Load(), "expecting a load");
 1649   LoadNode *ld = n->as_Load();
 1650   return ld->is_acquire();
 1651 }
 1652 
 1653 bool unnecessary_release(const Node *n)
 1654 {
 1655   assert((n->is_MemBar() &&
 1656           n->Opcode() == Op_MemBarRelease),
 1657          "expecting a release membar");
 1658 
 1659   MemBarNode *barrier = n->as_MemBar();
 1660   if (!barrier->leading()) {
 1661     return false;
 1662   } else {
 1663     Node* trailing = barrier->trailing_membar();
 1664     MemBarNode* trailing_mb = trailing->as_MemBar();
 1665     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1666     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1667 
 1668     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1669     if (mem->is_Store()) {
 1670       assert(mem->as_Store()->is_release(), "");
 1671       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1672       return true;
 1673     } else {
 1674       assert(mem->is_LoadStore(), "");
 1675       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1676       return is_CAS(mem->Opcode(), true);
 1677     }
 1678   }
 1679   return false;
 1680 }
 1681 
 1682 bool unnecessary_volatile(const Node *n)
 1683 {
 1684   // assert n->is_MemBar();
 1685   MemBarNode *mbvol = n->as_MemBar();
 1686 
 1687   bool release = mbvol->trailing_store();
 1688   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1689 #ifdef ASSERT
 1690   if (release) {
 1691     Node* leading = mbvol->leading_membar();
 1692     assert(leading->Opcode() == Op_MemBarRelease, "");
 1693     assert(leading->as_MemBar()->leading_store(), "");
 1694     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1695   }
 1696 #endif
 1697 
 1698   return release;
 1699 }
 1700 
 1701 // predicates controlling emit of str<x>/stlr<x>
 1702 
 1703 bool needs_releasing_store(const Node *n)
 1704 {
 1705   // assert n->is_Store();
 1706   StoreNode *st = n->as_Store();
 1707   return st->trailing_membar() != NULL;
 1708 }
 1709 
 1710 // predicate controlling translation of CAS
 1711 //
 1712 // returns true if CAS needs to use an acquiring load otherwise false
 1713 
 1714 bool needs_acquiring_load_exclusive(const Node *n)
 1715 {
 1716   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1717   LoadStoreNode* ldst = n->as_LoadStore();
 1718   if (is_CAS(n->Opcode(), false)) {
 1719     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1720   } else {
 1721     return ldst->trailing_membar() != NULL;
 1722   }
 1723 
 1724   // so we can just return true here
 1725   return true;
 1726 }
 1727 
 1728 #define __ _masm.
 1729 
 1730 // advance declarations for helper functions to convert register
 1731 // indices to register objects
 1732 
 1733 // the ad file has to provide implementations of certain methods
 1734 // expected by the generic code
 1735 //
 1736 // REQUIRED FUNCTIONALITY
 1737 
 1738 //=============================================================================
 1739 
 1740 // !!!!! Special hack to get all types of calls to specify the byte offset
 1741 //       from the start of the call to the point where the return address
 1742 //       will point.
 1743 
 1744 int MachCallStaticJavaNode::ret_addr_offset()
 1745 {
 1746   // call should be a simple bl
 1747   int off = 4;
 1748   return off;
 1749 }
 1750 
 1751 int MachCallDynamicJavaNode::ret_addr_offset()
 1752 {
 1753   return 16; // movz, movk, movk, bl
 1754 }
 1755 
 1756 int MachCallRuntimeNode::ret_addr_offset() {
 1757   // for generated stubs the call will be
 1758   //   bl(addr)
 1759   // or with far branches
 1760   //   bl(trampoline_stub)
 1761   // for real runtime callouts it will be six instructions
 1762   // see aarch64_enc_java_to_runtime
 1763   //   adr(rscratch2, retaddr)
 1764   //   lea(rscratch1, RuntimeAddress(addr)
 1765   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1766   //   blr(rscratch1)
 1767   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1768   if (cb) {
 1769     return 1 * NativeInstruction::instruction_size;
 1770   } else {
 1771     return 6 * NativeInstruction::instruction_size;
 1772   }
 1773 }
 1774 
 1775 int MachCallNativeNode::ret_addr_offset() {
 1776   // This is implemented using aarch64_enc_java_to_runtime as above.
 1777   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1778   if (cb) {
 1779     return 1 * NativeInstruction::instruction_size;
 1780   } else {
 1781     return 6 * NativeInstruction::instruction_size;
 1782   }
 1783 }
 1784 
 1785 //=============================================================================
 1786 
 1787 #ifndef PRODUCT
 1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1789   st->print("BREAKPOINT");
 1790 }
 1791 #endif
 1792 
 1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1794   C2_MacroAssembler _masm(&cbuf);
 1795   __ brk(0);
 1796 }
 1797 
 1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1799   return MachNode::size(ra_);
 1800 }
 1801 
 1802 //=============================================================================
 1803 
 1804 #ifndef PRODUCT
 1805   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1806     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1807   }
 1808 #endif
 1809 
 1810   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1811     C2_MacroAssembler _masm(&cbuf);
 1812     for (int i = 0; i < _count; i++) {
 1813       __ nop();
 1814     }
 1815   }
 1816 
 1817   uint MachNopNode::size(PhaseRegAlloc*) const {
 1818     return _count * NativeInstruction::instruction_size;
 1819   }
 1820 
 1821 //=============================================================================
 1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1823 
 1824 int ConstantTable::calculate_table_base_offset() const {
 1825   return 0;  // absolute addressing, no offset
 1826 }
 1827 
 1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1830   ShouldNotReachHere();
 1831 }
 1832 
 1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1834   // Empty encoding
 1835 }
 1836 
 1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1838   return 0;
 1839 }
 1840 
 1841 #ifndef PRODUCT
 1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1843   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1844 }
 1845 #endif
 1846 
 1847 #ifndef PRODUCT
 1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1849   Compile* C = ra_->C;
 1850 
 1851   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1852 
 1853   if (C->output()->need_stack_bang(framesize))
 1854     st->print("# stack bang size=%d\n\t", framesize);
 1855 
 1856   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1857     st->print("sub  sp, sp, #%d\n\t", framesize);
 1858     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1859     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1860   } else {
 1861     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1862     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1863     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1864     st->print("sub  sp, sp, rscratch1");
 1865   }
 1866   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1867     st->print("\n\t");
 1868     st->print("ldr  rscratch1, [guard]\n\t");
 1869     st->print("dmb ishld\n\t");
 1870     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1871     st->print("cmp  rscratch1, rscratch2\n\t");
 1872     st->print("b.eq skip");
 1873     st->print("\n\t");
 1874     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1875     st->print("b skip\n\t");
 1876     st->print("guard: int\n\t");
 1877     st->print("\n\t");
 1878     st->print("skip:\n\t");
 1879   }
 1880 }
 1881 #endif
 1882 
 1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1884   Compile* C = ra_->C;
 1885   C2_MacroAssembler _masm(&cbuf);
 1886 
 1887   // n.b. frame size includes space for return pc and rfp
 1888   const int framesize = C->output()->frame_size_in_bytes();
 1889 
 1890   // insert a nop at the start of the prolog so we can patch in a
 1891   // branch if we need to invalidate the method later
 1892   __ nop();
 1893 
 1894   if (C->clinit_barrier_on_entry()) {
 1895     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1896 
 1897     Label L_skip_barrier;
 1898 
 1899     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1900     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1901     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1902     __ bind(L_skip_barrier);
 1903   }
 1904 
 1905   if (C->max_vector_size() > 0) {
 1906     __ reinitialize_ptrue();
 1907   }
 1908 
 1909   int bangsize = C->output()->bang_size_in_bytes();
 1910   if (C->output()->need_stack_bang(bangsize))
 1911     __ generate_stack_overflow_check(bangsize);
 1912 
 1913   __ build_frame(framesize);
 1914 
 1915   if (C->stub_function() == NULL) {
 1916     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1917     bs->nmethod_entry_barrier(&_masm);
 1918   }
 1919 
 1920   if (VerifyStackAtCalls) {
 1921     Unimplemented();
 1922   }
 1923 
 1924   C->output()->set_frame_complete(cbuf.insts_size());
 1925 
 1926   if (C->has_mach_constant_base_node()) {
 1927     // NOTE: We set the table base offset here because users might be
 1928     // emitted before MachConstantBaseNode.
 1929     ConstantTable& constant_table = C->output()->constant_table();
 1930     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1931   }
 1932 }
 1933 
 1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1935 {
 1936   return MachNode::size(ra_); // too many variables; just compute it
 1937                               // the hard way
 1938 }
 1939 
 1940 int MachPrologNode::reloc() const
 1941 {
 1942   return 0;
 1943 }
 1944 
 1945 //=============================================================================
 1946 
 1947 #ifndef PRODUCT
 1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1949   Compile* C = ra_->C;
 1950   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1951 
 1952   st->print("# pop frame %d\n\t",framesize);
 1953 
 1954   if (framesize == 0) {
 1955     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1956   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1957     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1958     st->print("add  sp, sp, #%d\n\t", framesize);
 1959   } else {
 1960     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, rscratch1\n\t");
 1962     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1963   }
 1964 
 1965   if (do_polling() && C->is_method_compilation()) {
 1966     st->print("# test polling word\n\t");
 1967     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1968     st->print("cmp  sp, rscratch1\n\t");
 1969     st->print("bhi #slow_path");
 1970   }
 1971 }
 1972 #endif
 1973 
 1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1975   Compile* C = ra_->C;
 1976   C2_MacroAssembler _masm(&cbuf);
 1977   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1978 
 1979   __ remove_frame(framesize);
 1980 
 1981   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1982     __ reserved_stack_check();
 1983   }
 1984 
 1985   if (do_polling() && C->is_method_compilation()) {
 1986     Label dummy_label;
 1987     Label* code_stub = &dummy_label;
 1988     if (!C->output()->in_scratch_emit_size()) {
 1989       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1990     }
 1991     __ relocate(relocInfo::poll_return_type);
 1992     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1993   }
 1994 }
 1995 
 1996 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1997   // Variable size. Determine dynamically.
 1998   return MachNode::size(ra_);
 1999 }
 2000 
 2001 int MachEpilogNode::reloc() const {
 2002   // Return number of relocatable values contained in this instruction.
 2003   return 1; // 1 for polling page.
 2004 }
 2005 
 2006 const Pipeline * MachEpilogNode::pipeline() const {
 2007   return MachNode::pipeline_class();
 2008 }
 2009 
 2010 //=============================================================================
 2011 
 2012 // Figure out which register class each belongs in: rc_int, rc_float or
 2013 // rc_stack.
 2014 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2015 
 2016 static enum RC rc_class(OptoReg::Name reg) {
 2017 
 2018   if (reg == OptoReg::Bad) {
 2019     return rc_bad;
 2020   }
 2021 
 2022   // we have 32 int registers * 2 halves
 2023   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2024 
 2025   if (reg < slots_of_int_registers) {
 2026     return rc_int;
 2027   }
 2028 
 2029   // we have 32 float register * 8 halves
 2030   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2031   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2032     return rc_float;
 2033   }
 2034 
 2035   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2036   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2037     return rc_predicate;
 2038   }
 2039 
 2040   // Between predicate regs & stack is the flags.
 2041   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2042 
 2043   return rc_stack;
 2044 }
 2045 
 2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2047   Compile* C = ra_->C;
 2048 
 2049   // Get registers to move.
 2050   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2051   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2052   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2053   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2054 
 2055   enum RC src_hi_rc = rc_class(src_hi);
 2056   enum RC src_lo_rc = rc_class(src_lo);
 2057   enum RC dst_hi_rc = rc_class(dst_hi);
 2058   enum RC dst_lo_rc = rc_class(dst_lo);
 2059 
 2060   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2061 
 2062   if (src_hi != OptoReg::Bad) {
 2063     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2064            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2065            "expected aligned-adjacent pairs");
 2066   }
 2067 
 2068   if (src_lo == dst_lo && src_hi == dst_hi) {
 2069     return 0;            // Self copy, no move.
 2070   }
 2071 
 2072   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2073               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2074   int src_offset = ra_->reg2offset(src_lo);
 2075   int dst_offset = ra_->reg2offset(dst_lo);
 2076 
 2077   if (bottom_type()->isa_vect() != NULL) {
 2078     uint ireg = ideal_reg();
 2079     if (ireg == Op_VecA && cbuf) {
 2080       C2_MacroAssembler _masm(cbuf);
 2081       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2082       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2083         // stack->stack
 2084         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2085                                                 sve_vector_reg_size_in_bytes);
 2086       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2087         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2088                             sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2090         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2091                               sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2093         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2094                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2095                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2096       } else {
 2097         ShouldNotReachHere();
 2098       }
 2099     } else if (cbuf) {
 2100       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2101       C2_MacroAssembler _masm(cbuf);
 2102       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2103       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2104         // stack->stack
 2105         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2106         if (ireg == Op_VecD) {
 2107           __ unspill(rscratch1, true, src_offset);
 2108           __ spill(rscratch1, true, dst_offset);
 2109         } else {
 2110           __ spill_copy128(src_offset, dst_offset);
 2111         }
 2112       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2113         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2114                ireg == Op_VecD ? __ T8B : __ T16B,
 2115                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2116       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2117         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2118                  ireg == Op_VecD ? __ D : __ Q,
 2119                  ra_->reg2offset(dst_lo));
 2120       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2121         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2122                    ireg == Op_VecD ? __ D : __ Q,
 2123                    ra_->reg2offset(src_lo));
 2124       } else {
 2125         ShouldNotReachHere();
 2126       }
 2127     }
 2128   } else if (cbuf) {
 2129     C2_MacroAssembler _masm(cbuf);
 2130     switch (src_lo_rc) {
 2131     case rc_int:
 2132       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2133         if (is64) {
 2134             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2135                    as_Register(Matcher::_regEncode[src_lo]));
 2136         } else {
 2137             C2_MacroAssembler _masm(cbuf);
 2138             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2139                     as_Register(Matcher::_regEncode[src_lo]));
 2140         }
 2141       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2142         if (is64) {
 2143             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2144                      as_Register(Matcher::_regEncode[src_lo]));
 2145         } else {
 2146             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2147                      as_Register(Matcher::_regEncode[src_lo]));
 2148         }
 2149       } else {                    // gpr --> stack spill
 2150         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2151         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2152       }
 2153       break;
 2154     case rc_float:
 2155       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2156         if (is64) {
 2157             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2158                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2159         } else {
 2160             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2161                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2162         }
 2163       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2164         if (is64) {
 2165             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2166                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2167         } else {
 2168             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         }
 2171       } else {                    // fpr --> stack spill
 2172         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2173         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2174                  is64 ? __ D : __ S, dst_offset);
 2175       }
 2176       break;
 2177     case rc_stack:
 2178       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2179         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2180       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2181         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2182                    is64 ? __ D : __ S, src_offset);
 2183       } else {                    // stack --> stack copy
 2184         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2185         __ unspill(rscratch1, is64, src_offset);
 2186         __ spill(rscratch1, is64, dst_offset);
 2187       }
 2188       break;
 2189     default:
 2190       assert(false, "bad rc_class for spill");
 2191       ShouldNotReachHere();
 2192     }
 2193   }
 2194 
 2195   if (st) {
 2196     st->print("spill ");
 2197     if (src_lo_rc == rc_stack) {
 2198       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2199     } else {
 2200       st->print("%s -> ", Matcher::regName[src_lo]);
 2201     }
 2202     if (dst_lo_rc == rc_stack) {
 2203       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2204     } else {
 2205       st->print("%s", Matcher::regName[dst_lo]);
 2206     }
 2207     if (bottom_type()->isa_vect() != NULL) {
 2208       int vsize = 0;
 2209       switch (ideal_reg()) {
 2210       case Op_VecD:
 2211         vsize = 64;
 2212         break;
 2213       case Op_VecX:
 2214         vsize = 128;
 2215         break;
 2216       case Op_VecA:
 2217         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2218         break;
 2219       default:
 2220         assert(false, "bad register type for spill");
 2221         ShouldNotReachHere();
 2222       }
 2223       st->print("\t# vector spill size = %d", vsize);
 2224     } else {
 2225       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2226     }
 2227   }
 2228 
 2229   return 0;
 2230 
 2231 }
 2232 
 2233 #ifndef PRODUCT
 2234 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2235   if (!ra_)
 2236     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2237   else
 2238     implementation(NULL, ra_, false, st);
 2239 }
 2240 #endif
 2241 
 2242 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2243   implementation(&cbuf, ra_, false, NULL);
 2244 }
 2245 
 2246 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2247   return MachNode::size(ra_);
 2248 }
 2249 
 2250 //=============================================================================
 2251 
 2252 #ifndef PRODUCT
 2253 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2254   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2255   int reg = ra_->get_reg_first(this);
 2256   st->print("add %s, rsp, #%d]\t# box lock",
 2257             Matcher::regName[reg], offset);
 2258 }
 2259 #endif
 2260 
 2261 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2262   C2_MacroAssembler _masm(&cbuf);
 2263 
 2264   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2265   int reg    = ra_->get_encode(this);
 2266 
 2267   // This add will handle any 24-bit signed offset. 24 bits allows an
 2268   // 8 megabyte stack frame.
 2269   __ add(as_Register(reg), sp, offset);
 2270 }
 2271 
 2272 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2273   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2274   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2275 
 2276   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2277     return NativeInstruction::instruction_size;
 2278   } else {
 2279     return 2 * NativeInstruction::instruction_size;
 2280   }
 2281 }
 2282 
 2283 //=============================================================================
 2284 
 2285 #ifndef PRODUCT
 2286 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2287 {
 2288   st->print_cr("# MachUEPNode");
 2289   if (UseCompressedClassPointers) {
 2290     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2291     if (CompressedKlassPointers::shift() != 0) {
 2292       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2293     }
 2294   } else {
 2295    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2296   }
 2297   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2298   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2299 }
 2300 #endif
 2301 
 2302 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2303 {
 2304   // This is the unverified entry point.
 2305   C2_MacroAssembler _masm(&cbuf);
 2306 
 2307   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2308   Label skip;
 2309   // TODO
 2310   // can we avoid this skip and still use a reloc?
 2311   __ br(Assembler::EQ, skip);
 2312   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2313   __ bind(skip);
 2314 }
 2315 
 2316 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2317 {
 2318   return MachNode::size(ra_);
 2319 }
 2320 
 2321 // REQUIRED EMIT CODE
 2322 
 2323 //=============================================================================
 2324 
 2325 // Emit exception handler code.
 2326 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2327 {
 2328   // mov rscratch1 #exception_blob_entry_point
 2329   // br rscratch1
 2330   // Note that the code buffer's insts_mark is always relative to insts.
 2331   // That's why we must use the macroassembler to generate a handler.
 2332   C2_MacroAssembler _masm(&cbuf);
 2333   address base = __ start_a_stub(size_exception_handler());
 2334   if (base == NULL) {
 2335     ciEnv::current()->record_failure("CodeCache is full");
 2336     return 0;  // CodeBuffer::expand failed
 2337   }
 2338   int offset = __ offset();
 2339   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2340   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2341   __ end_a_stub();
 2342   return offset;
 2343 }
 2344 
 2345 // Emit deopt handler code.
 2346 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2347 {
 2348   // Note that the code buffer's insts_mark is always relative to insts.
 2349   // That's why we must use the macroassembler to generate a handler.
 2350   C2_MacroAssembler _masm(&cbuf);
 2351   address base = __ start_a_stub(size_deopt_handler());
 2352   if (base == NULL) {
 2353     ciEnv::current()->record_failure("CodeCache is full");
 2354     return 0;  // CodeBuffer::expand failed
 2355   }
 2356   int offset = __ offset();
 2357 
 2358   __ adr(lr, __ pc());
 2359   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2360 
 2361   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
 2362   __ end_a_stub();
 2363   return offset;
 2364 }
 2365 
 2366 // REQUIRED MATCHER CODE
 2367 
 2368 //=============================================================================
 2369 
 2370 const bool Matcher::match_rule_supported(int opcode) {
 2371   if (!has_match_rule(opcode))
 2372     return false;
 2373 
 2374   bool ret_value = true;
 2375   switch (opcode) {
 2376     case Op_CacheWB:
 2377     case Op_CacheWBPreSync:
 2378     case Op_CacheWBPostSync:
 2379       if (!VM_Version::supports_data_cache_line_flush()) {
 2380         ret_value = false;
 2381       }
 2382       break;
 2383   }
 2384 
 2385   return ret_value; // Per default match rules are supported.
 2386 }
 2387 
 2388 // Identify extra cases that we might want to provide match rules for vector nodes and
 2389 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2390 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2391   if (!match_rule_supported(opcode)) {
 2392     return false;
 2393   }
 2394   int bit_size = vlen * type2aelembytes(bt) * 8;
 2395   if (UseSVE == 0 && bit_size > 128) {
 2396     return false;
 2397   }
 2398   if (UseSVE > 0) {
 2399     return op_sve_supported(opcode, vlen, bt);
 2400   } else { // NEON
 2401     // Special cases
 2402     switch (opcode) {
 2403     case Op_VectorMaskCmp:
 2404     // We don't have VectorReinterpret with bit_size less than 64 support for
 2405     // now, even for byte type. To be refined with fully VectorCast support.
 2406     case Op_VectorReinterpret:
 2407       if (vlen < 2 || bit_size < 64) {
 2408         return false;
 2409       }
 2410       break;
 2411     case Op_MulAddVS2VI:
 2412       if (bit_size < 128) {
 2413         return false;
 2414       }
 2415       break;
 2416     case Op_MulVL:
 2417       return false;
 2418     case Op_VectorLoadShuffle:
 2419     case Op_VectorRearrange:
 2420       if (vlen < 4) {
 2421         return false;
 2422       }
 2423       break;
 2424     // Some types of VectorCast are not implemented for now.
 2425     case Op_VectorCastI2X:
 2426       if (bt == T_BYTE) {
 2427         return false;
 2428       }
 2429       break;
 2430     case Op_VectorCastS2X:
 2431       if (vlen < 4 || bit_size < 64) {
 2432         return false;
 2433       }
 2434       break;
 2435     case Op_VectorCastF2X:
 2436     case Op_VectorCastD2X:
 2437       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2438         return false;
 2439       }
 2440       break;
 2441     case Op_LoadVectorGather:
 2442     case Op_StoreVectorScatter:
 2443       return false;
 2444     default:
 2445       break;
 2446     }
 2447   }
 2448   return vector_size_supported(bt, vlen);
 2449 }
 2450 
 2451 const RegMask* Matcher::predicate_reg_mask(void) {
 2452   return &_PR_REG_mask;
 2453 }
 2454 
 2455 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2456   return new TypeVectMask(elemTy, length);
 2457 }
 2458 
 2459 // Vector calling convention not yet implemented.
 2460 const bool Matcher::supports_vector_calling_convention(void) {
 2461   return false;
 2462 }
 2463 
 2464 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2465   Unimplemented();
 2466   return OptoRegPair(0, 0);
 2467 }
 2468 
 2469 // Is this branch offset short enough that a short branch can be used?
 2470 //
 2471 // NOTE: If the platform does not provide any short branch variants, then
 2472 //       this method should return false for offset 0.
 2473 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2474   // The passed offset is relative to address of the branch.
 2475 
 2476   return (-32768 <= offset && offset < 32768);
 2477 }
 2478 
 2479 // Vector width in bytes.
 2480 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2481   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2482   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2483   // Minimum 2 values in vector
 2484   if (size < 2*type2aelembytes(bt)) size = 0;
 2485   // But never < 4
 2486   if (size < 4) size = 0;
 2487   return size;
 2488 }
 2489 
 2490 // Limits on vector size (number of elements) loaded into vector.
 2491 const int Matcher::max_vector_size(const BasicType bt) {
 2492   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2493 }
 2494 
 2495 const int Matcher::min_vector_size(const BasicType bt) {
 2496   int max_size = max_vector_size(bt);
 2497   // Limit the min vector size to 8 bytes.
 2498   int size = 8 / type2aelembytes(bt);
 2499   if (bt == T_BYTE) {
 2500     // To support vector api shuffle/rearrange.
 2501     size = 4;
 2502   } else if (bt == T_BOOLEAN) {
 2503     // To support vector api load/store mask.
 2504     size = 2;
 2505   }
 2506   if (size < 2) size = 2;
 2507   return MIN2(size, max_size);
 2508 }
 2509 
 2510 // Actual max scalable vector register length.
 2511 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2512   return Matcher::max_vector_size(bt);
 2513 }
 2514 
 2515 // Vector ideal reg.
 2516 const uint Matcher::vector_ideal_reg(int len) {
 2517   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2518     return Op_VecA;
 2519   }
 2520   switch(len) {
 2521     // For 16-bit/32-bit mask vector, reuse VecD.
 2522     case  2:
 2523     case  4:
 2524     case  8: return Op_VecD;
 2525     case 16: return Op_VecX;
 2526   }
 2527   ShouldNotReachHere();
 2528   return 0;
 2529 }
 2530 
 2531 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2532   ShouldNotReachHere(); // generic vector operands not supported
 2533   return NULL;
 2534 }
 2535 
 2536 bool Matcher::is_reg2reg_move(MachNode* m) {
 2537   ShouldNotReachHere();  // generic vector operands not supported
 2538   return false;
 2539 }
 2540 
 2541 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2542   ShouldNotReachHere();  // generic vector operands not supported
 2543   return false;
 2544 }
 2545 
 2546 // Return whether or not this register is ever used as an argument.
 2547 // This function is used on startup to build the trampoline stubs in
 2548 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2549 // call in the trampoline, and arguments in those registers not be
 2550 // available to the callee.
 2551 bool Matcher::can_be_java_arg(int reg)
 2552 {
 2553   return
 2554     reg ==  R0_num || reg == R0_H_num ||
 2555     reg ==  R1_num || reg == R1_H_num ||
 2556     reg ==  R2_num || reg == R2_H_num ||
 2557     reg ==  R3_num || reg == R3_H_num ||
 2558     reg ==  R4_num || reg == R4_H_num ||
 2559     reg ==  R5_num || reg == R5_H_num ||
 2560     reg ==  R6_num || reg == R6_H_num ||
 2561     reg ==  R7_num || reg == R7_H_num ||
 2562     reg ==  V0_num || reg == V0_H_num ||
 2563     reg ==  V1_num || reg == V1_H_num ||
 2564     reg ==  V2_num || reg == V2_H_num ||
 2565     reg ==  V3_num || reg == V3_H_num ||
 2566     reg ==  V4_num || reg == V4_H_num ||
 2567     reg ==  V5_num || reg == V5_H_num ||
 2568     reg ==  V6_num || reg == V6_H_num ||
 2569     reg ==  V7_num || reg == V7_H_num;
 2570 }
 2571 
 2572 bool Matcher::is_spillable_arg(int reg)
 2573 {
 2574   return can_be_java_arg(reg);
 2575 }
 2576 
 2577 uint Matcher::int_pressure_limit()
 2578 {
 2579   // JDK-8183543: When taking the number of available registers as int
 2580   // register pressure threshold, the jtreg test:
 2581   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2582   // failed due to C2 compilation failure with
 2583   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2584   //
 2585   // A derived pointer is live at CallNode and then is flagged by RA
 2586   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2587   // derived pointers and lastly fail to spill after reaching maximum
 2588   // number of iterations. Lowering the default pressure threshold to
 2589   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2590   // a high register pressure area of the code so that split_DEF can
 2591   // generate DefinitionSpillCopy for the derived pointer.
 2592   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2593   if (!PreserveFramePointer) {
 2594     // When PreserveFramePointer is off, frame pointer is allocatable,
 2595     // but different from other SOC registers, it is excluded from
 2596     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2597     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2598     // See check_pressure_at_fatproj().
 2599     default_int_pressure_threshold--;
 2600   }
 2601   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2602 }
 2603 
 2604 uint Matcher::float_pressure_limit()
 2605 {
 2606   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2607   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2608 }
 2609 
 2610 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2611   return false;
 2612 }
 2613 
 2614 RegMask Matcher::divI_proj_mask() {
 2615   ShouldNotReachHere();
 2616   return RegMask();
 2617 }
 2618 
 2619 // Register for MODI projection of divmodI.
 2620 RegMask Matcher::modI_proj_mask() {
 2621   ShouldNotReachHere();
 2622   return RegMask();
 2623 }
 2624 
 2625 // Register for DIVL projection of divmodL.
 2626 RegMask Matcher::divL_proj_mask() {
 2627   ShouldNotReachHere();
 2628   return RegMask();
 2629 }
 2630 
 2631 // Register for MODL projection of divmodL.
 2632 RegMask Matcher::modL_proj_mask() {
 2633   ShouldNotReachHere();
 2634   return RegMask();
 2635 }
 2636 
 2637 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2638   return FP_REG_mask();
 2639 }
 2640 
 2641 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2642   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2643     Node* u = addp->fast_out(i);
 2644     if (u->is_LoadStore()) {
 2645       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2646       // instructions) only take register indirect as an operand, so
 2647       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2648       // must fail.
 2649       return false;
 2650     }
 2651     if (u->is_Mem()) {
 2652       int opsize = u->as_Mem()->memory_size();
 2653       assert(opsize > 0, "unexpected memory operand size");
 2654       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2655         return false;
 2656       }
 2657     }
 2658   }
 2659   return true;
 2660 }
 2661 
 2662 // Should the matcher clone input 'm' of node 'n'?
 2663 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2664   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
 2665     mstack.push(m, Visit);           // m = ShiftCntV
 2666     return true;
 2667   }
 2668   return false;
 2669 }
 2670 
 2671 // Should the Matcher clone shifts on addressing modes, expecting them
 2672 // to be subsumed into complex addressing expressions or compute them
 2673 // into registers?
 2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2675   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2676     return true;
 2677   }
 2678 
 2679   Node *off = m->in(AddPNode::Offset);
 2680   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2681       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2682       // Are there other uses besides address expressions?
 2683       !is_visited(off)) {
 2684     address_visited.set(off->_idx); // Flag as address_visited
 2685     mstack.push(off->in(2), Visit);
 2686     Node *conv = off->in(1);
 2687     if (conv->Opcode() == Op_ConvI2L &&
 2688         // Are there other uses besides address expressions?
 2689         !is_visited(conv)) {
 2690       address_visited.set(conv->_idx); // Flag as address_visited
 2691       mstack.push(conv->in(1), Pre_Visit);
 2692     } else {
 2693       mstack.push(conv, Pre_Visit);
 2694     }
 2695     address_visited.test_set(m->_idx); // Flag as address_visited
 2696     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2697     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2698     return true;
 2699   } else if (off->Opcode() == Op_ConvI2L &&
 2700              // Are there other uses besides address expressions?
 2701              !is_visited(off)) {
 2702     address_visited.test_set(m->_idx); // Flag as address_visited
 2703     address_visited.set(off->_idx); // Flag as address_visited
 2704     mstack.push(off->in(1), Pre_Visit);
 2705     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2706     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2707     return true;
 2708   }
 2709   return false;
 2710 }
 2711 
 2712 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2713   C2_MacroAssembler _masm(&cbuf);                                       \
 2714   {                                                                     \
 2715     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2716     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2717     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2718     __ INSN(REG, as_Register(BASE));                                    \
 2719   }
 2720 
 2721 
 2722 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2723   {
 2724     Address::extend scale;
 2725 
 2726     // Hooboy, this is fugly.  We need a way to communicate to the
 2727     // encoder that the index needs to be sign extended, so we have to
 2728     // enumerate all the cases.
 2729     switch (opcode) {
 2730     case INDINDEXSCALEDI2L:
 2731     case INDINDEXSCALEDI2LN:
 2732     case INDINDEXI2L:
 2733     case INDINDEXI2LN:
 2734       scale = Address::sxtw(size);
 2735       break;
 2736     default:
 2737       scale = Address::lsl(size);
 2738     }
 2739 
 2740     if (index == -1) {
 2741       return Address(base, disp);
 2742     } else {
 2743       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2744       return Address(base, as_Register(index), scale);
 2745     }
 2746   }
 2747 
 2748 
 2749 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2750 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2751 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2752 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2753                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2754 
 2755   // Used for all non-volatile memory accesses.  The use of
 2756   // $mem->opcode() to discover whether this pattern uses sign-extended
 2757   // offsets is something of a kludge.
 2758   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2759                         Register reg, int opcode,
 2760                         Register base, int index, int scale, int disp,
 2761                         int size_in_memory)
 2762   {
 2763     Address addr = mem2address(opcode, base, index, scale, disp);
 2764     if (addr.getMode() == Address::base_plus_offset) {
 2765       /* If we get an out-of-range offset it is a bug in the compiler,
 2766          so we assert here. */
 2767       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2768              "c2 compiler bug");
 2769       /* Fix up any out-of-range offsets. */
 2770       assert_different_registers(rscratch1, base);
 2771       assert_different_registers(rscratch1, reg);
 2772       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2773     }
 2774     (masm.*insn)(reg, addr);
 2775   }
 2776 
 2777   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2778                         FloatRegister reg, int opcode,
 2779                         Register base, int index, int size, int disp,
 2780                         int size_in_memory)
 2781   {
 2782     Address::extend scale;
 2783 
 2784     switch (opcode) {
 2785     case INDINDEXSCALEDI2L:
 2786     case INDINDEXSCALEDI2LN:
 2787       scale = Address::sxtw(size);
 2788       break;
 2789     default:
 2790       scale = Address::lsl(size);
 2791     }
 2792 
 2793     if (index == -1) {
 2794       /* If we get an out-of-range offset it is a bug in the compiler,
 2795          so we assert here. */
 2796       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2797       /* Fix up any out-of-range offsets. */
 2798       assert_different_registers(rscratch1, base);
 2799       Address addr = Address(base, disp);
 2800       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2801       (masm.*insn)(reg, addr);
 2802     } else {
 2803       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2804       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2805     }
 2806   }
 2807 
 2808   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2809                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2810                         int opcode, Register base, int index, int size, int disp)
 2811   {
 2812     if (index == -1) {
 2813       (masm.*insn)(reg, T, Address(base, disp));
 2814     } else {
 2815       assert(disp == 0, "unsupported address mode");
 2816       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2817     }
 2818   }
 2819 
 2820 %}
 2821 
 2822 
 2823 
 2824 //----------ENCODING BLOCK-----------------------------------------------------
 2825 // This block specifies the encoding classes used by the compiler to
 2826 // output byte streams.  Encoding classes are parameterized macros
 2827 // used by Machine Instruction Nodes in order to generate the bit
 2828 // encoding of the instruction.  Operands specify their base encoding
 2829 // interface with the interface keyword.  There are currently
 2830 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2831 // COND_INTER.  REG_INTER causes an operand to generate a function
 2832 // which returns its register number when queried.  CONST_INTER causes
 2833 // an operand to generate a function which returns the value of the
 2834 // constant when queried.  MEMORY_INTER causes an operand to generate
 2835 // four functions which return the Base Register, the Index Register,
 2836 // the Scale Value, and the Offset Value of the operand when queried.
 2837 // COND_INTER causes an operand to generate six functions which return
 2838 // the encoding code (ie - encoding bits for the instruction)
 2839 // associated with each basic boolean condition for a conditional
 2840 // instruction.
 2841 //
 2842 // Instructions specify two basic values for encoding.  Again, a
 2843 // function is available to check if the constant displacement is an
 2844 // oop. They use the ins_encode keyword to specify their encoding
 2845 // classes (which must be a sequence of enc_class names, and their
 2846 // parameters, specified in the encoding block), and they use the
 2847 // opcode keyword to specify, in order, their primary, secondary, and
 2848 // tertiary opcode.  Only the opcode sections which a particular
 2849 // instruction needs for encoding need to be specified.
 2850 encode %{
 2851   // Build emit functions for each basic byte or larger field in the
 2852   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2853   // from C++ code in the enc_class source block.  Emit functions will
 2854   // live in the main source block for now.  In future, we can
 2855   // generalize this by adding a syntax that specifies the sizes of
 2856   // fields in an order, so that the adlc can build the emit functions
 2857   // automagically
 2858 
 2859   // catch all for unimplemented encodings
 2860   enc_class enc_unimplemented %{
 2861     C2_MacroAssembler _masm(&cbuf);
 2862     __ unimplemented("C2 catch all");
 2863   %}
 2864 
 2865   // BEGIN Non-volatile memory access
 2866 
 2867   // This encoding class is generated automatically from ad_encode.m4.
 2868   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2869   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2870     Register dst_reg = as_Register($dst$$reg);
 2871     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2872                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2873   %}
 2874 
 2875   // This encoding class is generated automatically from ad_encode.m4.
 2876   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2877   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2878     Register dst_reg = as_Register($dst$$reg);
 2879     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2880                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2881   %}
 2882 
 2883   // This encoding class is generated automatically from ad_encode.m4.
 2884   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2885   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2886     Register dst_reg = as_Register($dst$$reg);
 2887     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2888                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2889   %}
 2890 
 2891   // This encoding class is generated automatically from ad_encode.m4.
 2892   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2893   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2894     Register dst_reg = as_Register($dst$$reg);
 2895     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2896                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2897   %}
 2898 
 2899   // This encoding class is generated automatically from ad_encode.m4.
 2900   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2901   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2902     Register dst_reg = as_Register($dst$$reg);
 2903     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2904                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2905   %}
 2906 
 2907   // This encoding class is generated automatically from ad_encode.m4.
 2908   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2909   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2910     Register dst_reg = as_Register($dst$$reg);
 2911     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2912                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2913   %}
 2914 
 2915   // This encoding class is generated automatically from ad_encode.m4.
 2916   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2917   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2918     Register dst_reg = as_Register($dst$$reg);
 2919     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2921   %}
 2922 
 2923   // This encoding class is generated automatically from ad_encode.m4.
 2924   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2925   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2926     Register dst_reg = as_Register($dst$$reg);
 2927     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2928                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2929   %}
 2930 
 2931   // This encoding class is generated automatically from ad_encode.m4.
 2932   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2933   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2934     Register dst_reg = as_Register($dst$$reg);
 2935     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2936                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2937   %}
 2938 
 2939   // This encoding class is generated automatically from ad_encode.m4.
 2940   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2941   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2942     Register dst_reg = as_Register($dst$$reg);
 2943     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2945   %}
 2946 
 2947   // This encoding class is generated automatically from ad_encode.m4.
 2948   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2949   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2950     Register dst_reg = as_Register($dst$$reg);
 2951     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2952                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2953   %}
 2954 
 2955   // This encoding class is generated automatically from ad_encode.m4.
 2956   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2957   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2958     Register dst_reg = as_Register($dst$$reg);
 2959     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2960                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2961   %}
 2962 
 2963   // This encoding class is generated automatically from ad_encode.m4.
 2964   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2965   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2966     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2967     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2968                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2969   %}
 2970 
 2971   // This encoding class is generated automatically from ad_encode.m4.
 2972   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2973   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2974     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2975     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2976                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2977   %}
 2978 
 2979   // This encoding class is generated automatically from ad_encode.m4.
 2980   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2981   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2982     Register src_reg = as_Register($src$$reg);
 2983     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2984                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2985   %}
 2986 
 2987   // This encoding class is generated automatically from ad_encode.m4.
 2988   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2989   enc_class aarch64_enc_strb0(memory1 mem) %{
 2990     C2_MacroAssembler _masm(&cbuf);
 2991     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2992                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2993   %}
 2994 
 2995   // This encoding class is generated automatically from ad_encode.m4.
 2996   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2997   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2998     Register src_reg = as_Register($src$$reg);
 2999     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3000                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3001   %}
 3002 
 3003   // This encoding class is generated automatically from ad_encode.m4.
 3004   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3005   enc_class aarch64_enc_strh0(memory2 mem) %{
 3006     C2_MacroAssembler _masm(&cbuf);
 3007     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3008                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3009   %}
 3010 
 3011   // This encoding class is generated automatically from ad_encode.m4.
 3012   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3013   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3014     Register src_reg = as_Register($src$$reg);
 3015     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3016                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3017   %}
 3018 
 3019   // This encoding class is generated automatically from ad_encode.m4.
 3020   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3021   enc_class aarch64_enc_strw0(memory4 mem) %{
 3022     C2_MacroAssembler _masm(&cbuf);
 3023     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3024                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3025   %}
 3026 
 3027   // This encoding class is generated automatically from ad_encode.m4.
 3028   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3029   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3030     Register src_reg = as_Register($src$$reg);
 3031     // we sometimes get asked to store the stack pointer into the
 3032     // current thread -- we cannot do that directly on AArch64
 3033     if (src_reg == r31_sp) {
 3034       C2_MacroAssembler _masm(&cbuf);
 3035       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3036       __ mov(rscratch2, sp);
 3037       src_reg = rscratch2;
 3038     }
 3039     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3041   %}
 3042 
 3043   // This encoding class is generated automatically from ad_encode.m4.
 3044   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3045   enc_class aarch64_enc_str0(memory8 mem) %{
 3046     C2_MacroAssembler _masm(&cbuf);
 3047     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3048                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3049   %}
 3050 
 3051   // This encoding class is generated automatically from ad_encode.m4.
 3052   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3053   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3054     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3055     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3056                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3057   %}
 3058 
 3059   // This encoding class is generated automatically from ad_encode.m4.
 3060   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3061   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3062     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3063     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3064                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3065   %}
 3066 
 3067   // This encoding class is generated automatically from ad_encode.m4.
 3068   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3069   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3070       C2_MacroAssembler _masm(&cbuf);
 3071       __ membar(Assembler::StoreStore);
 3072       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3073                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3074   %}
 3075 
 3076   // END Non-volatile memory access
 3077 
 3078   // Vector loads and stores
 3079   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3080     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3081     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3082        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3083   %}
 3084 
 3085   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3086     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3087     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3088        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3089   %}
 3090 
 3091   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3092     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3093     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3094        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3095   %}
 3096 
 3097   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3098     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3099     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3100        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3101   %}
 3102 
 3103   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3104     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3105     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3106        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3107   %}
 3108 
 3109   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3110     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3111     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3112        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3113   %}
 3114 
 3115   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3116     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3117     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3118        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3122     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3123     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3124        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3125   %}
 3126 
 3127   // volatile loads and stores
 3128 
 3129   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3130     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3131                  rscratch1, stlrb);
 3132   %}
 3133 
 3134   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3135     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136                  rscratch1, stlrh);
 3137   %}
 3138 
 3139   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3140     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3141                  rscratch1, stlrw);
 3142   %}
 3143 
 3144 
 3145   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3146     Register dst_reg = as_Register($dst$$reg);
 3147     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3148              rscratch1, ldarb);
 3149     __ sxtbw(dst_reg, dst_reg);
 3150   %}
 3151 
 3152   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3153     Register dst_reg = as_Register($dst$$reg);
 3154     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3155              rscratch1, ldarb);
 3156     __ sxtb(dst_reg, dst_reg);
 3157   %}
 3158 
 3159   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3160     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3161              rscratch1, ldarb);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3165     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3166              rscratch1, ldarb);
 3167   %}
 3168 
 3169   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3170     Register dst_reg = as_Register($dst$$reg);
 3171     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3172              rscratch1, ldarh);
 3173     __ sxthw(dst_reg, dst_reg);
 3174   %}
 3175 
 3176   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3177     Register dst_reg = as_Register($dst$$reg);
 3178     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3179              rscratch1, ldarh);
 3180     __ sxth(dst_reg, dst_reg);
 3181   %}
 3182 
 3183   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3184     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3185              rscratch1, ldarh);
 3186   %}
 3187 
 3188   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3189     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3190              rscratch1, ldarh);
 3191   %}
 3192 
 3193   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3194     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3195              rscratch1, ldarw);
 3196   %}
 3197 
 3198   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3199     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3200              rscratch1, ldarw);
 3201   %}
 3202 
 3203   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3204     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3205              rscratch1, ldar);
 3206   %}
 3207 
 3208   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3209     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3210              rscratch1, ldarw);
 3211     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3212   %}
 3213 
 3214   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3215     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3216              rscratch1, ldar);
 3217     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3218   %}
 3219 
 3220   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3221     Register src_reg = as_Register($src$$reg);
 3222     // we sometimes get asked to store the stack pointer into the
 3223     // current thread -- we cannot do that directly on AArch64
 3224     if (src_reg == r31_sp) {
 3225       C2_MacroAssembler _masm(&cbuf);
 3226       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3227       __ mov(rscratch2, sp);
 3228       src_reg = rscratch2;
 3229     }
 3230     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3231                  rscratch1, stlr);
 3232   %}
 3233 
 3234   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3235     {
 3236       C2_MacroAssembler _masm(&cbuf);
 3237       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3238       __ fmovs(rscratch2, src_reg);
 3239     }
 3240     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3241                  rscratch1, stlrw);
 3242   %}
 3243 
 3244   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3245     {
 3246       C2_MacroAssembler _masm(&cbuf);
 3247       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3248       __ fmovd(rscratch2, src_reg);
 3249     }
 3250     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3251                  rscratch1, stlr);
 3252   %}
 3253 
 3254   // synchronized read/update encodings
 3255 
 3256   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3257     C2_MacroAssembler _masm(&cbuf);
 3258     Register dst_reg = as_Register($dst$$reg);
 3259     Register base = as_Register($mem$$base);
 3260     int index = $mem$$index;
 3261     int scale = $mem$$scale;
 3262     int disp = $mem$$disp;
 3263     if (index == -1) {
 3264        if (disp != 0) {
 3265         __ lea(rscratch1, Address(base, disp));
 3266         __ ldaxr(dst_reg, rscratch1);
 3267       } else {
 3268         // TODO
 3269         // should we ever get anything other than this case?
 3270         __ ldaxr(dst_reg, base);
 3271       }
 3272     } else {
 3273       Register index_reg = as_Register(index);
 3274       if (disp == 0) {
 3275         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3276         __ ldaxr(dst_reg, rscratch1);
 3277       } else {
 3278         __ lea(rscratch1, Address(base, disp));
 3279         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3280         __ ldaxr(dst_reg, rscratch1);
 3281       }
 3282     }
 3283   %}
 3284 
 3285   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3286     C2_MacroAssembler _masm(&cbuf);
 3287     Register src_reg = as_Register($src$$reg);
 3288     Register base = as_Register($mem$$base);
 3289     int index = $mem$$index;
 3290     int scale = $mem$$scale;
 3291     int disp = $mem$$disp;
 3292     if (index == -1) {
 3293        if (disp != 0) {
 3294         __ lea(rscratch2, Address(base, disp));
 3295         __ stlxr(rscratch1, src_reg, rscratch2);
 3296       } else {
 3297         // TODO
 3298         // should we ever get anything other than this case?
 3299         __ stlxr(rscratch1, src_reg, base);
 3300       }
 3301     } else {
 3302       Register index_reg = as_Register(index);
 3303       if (disp == 0) {
 3304         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3305         __ stlxr(rscratch1, src_reg, rscratch2);
 3306       } else {
 3307         __ lea(rscratch2, Address(base, disp));
 3308         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3309         __ stlxr(rscratch1, src_reg, rscratch2);
 3310       }
 3311     }
 3312     __ cmpw(rscratch1, zr);
 3313   %}
 3314 
 3315   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3316     C2_MacroAssembler _masm(&cbuf);
 3317     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3318     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3319                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3320                /*weak*/ false, noreg);
 3321   %}
 3322 
 3323   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3324     C2_MacroAssembler _masm(&cbuf);
 3325     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3326     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3327                Assembler::word, /*acquire*/ false, /*release*/ true,
 3328                /*weak*/ false, noreg);
 3329   %}
 3330 
 3331   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3332     C2_MacroAssembler _masm(&cbuf);
 3333     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3334     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3335                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3336                /*weak*/ false, noreg);
 3337   %}
 3338 
 3339   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3340     C2_MacroAssembler _masm(&cbuf);
 3341     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3342     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3343                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3344                /*weak*/ false, noreg);
 3345   %}
 3346 
 3347 
 3348   // The only difference between aarch64_enc_cmpxchg and
 3349   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3350   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3351   // lock.
 3352   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3353     C2_MacroAssembler _masm(&cbuf);
 3354     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3355     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3356                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3357                /*weak*/ false, noreg);
 3358   %}
 3359 
 3360   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3361     C2_MacroAssembler _masm(&cbuf);
 3362     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3363     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3364                Assembler::word, /*acquire*/ true, /*release*/ true,
 3365                /*weak*/ false, noreg);
 3366   %}
 3367 
 3368   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3369     C2_MacroAssembler _masm(&cbuf);
 3370     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3371     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3372                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3373                /*weak*/ false, noreg);
 3374   %}
 3375 
 3376   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3377     C2_MacroAssembler _masm(&cbuf);
 3378     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3379     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3380                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3381                /*weak*/ false, noreg);
 3382   %}
 3383 
 3384   // auxiliary used for CompareAndSwapX to set result register
 3385   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3386     C2_MacroAssembler _masm(&cbuf);
 3387     Register res_reg = as_Register($res$$reg);
 3388     __ cset(res_reg, Assembler::EQ);
 3389   %}
 3390 
 3391   // prefetch encodings
 3392 
 3393   enc_class aarch64_enc_prefetchw(memory mem) %{
 3394     C2_MacroAssembler _masm(&cbuf);
 3395     Register base = as_Register($mem$$base);
 3396     int index = $mem$$index;
 3397     int scale = $mem$$scale;
 3398     int disp = $mem$$disp;
 3399     if (index == -1) {
 3400       __ prfm(Address(base, disp), PSTL1KEEP);
 3401     } else {
 3402       Register index_reg = as_Register(index);
 3403       if (disp == 0) {
 3404         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3405       } else {
 3406         __ lea(rscratch1, Address(base, disp));
 3407 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3408       }
 3409     }
 3410   %}
 3411 
 3412   /// mov envcodings
 3413 
 3414   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3415     C2_MacroAssembler _masm(&cbuf);
 3416     uint32_t con = (uint32_t)$src$$constant;
 3417     Register dst_reg = as_Register($dst$$reg);
 3418     if (con == 0) {
 3419       __ movw(dst_reg, zr);
 3420     } else {
 3421       __ movw(dst_reg, con);
 3422     }
 3423   %}
 3424 
 3425   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3426     C2_MacroAssembler _masm(&cbuf);
 3427     Register dst_reg = as_Register($dst$$reg);
 3428     uint64_t con = (uint64_t)$src$$constant;
 3429     if (con == 0) {
 3430       __ mov(dst_reg, zr);
 3431     } else {
 3432       __ mov(dst_reg, con);
 3433     }
 3434   %}
 3435 
 3436   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3437     C2_MacroAssembler _masm(&cbuf);
 3438     Register dst_reg = as_Register($dst$$reg);
 3439     address con = (address)$src$$constant;
 3440     if (con == NULL || con == (address)1) {
 3441       ShouldNotReachHere();
 3442     } else {
 3443       relocInfo::relocType rtype = $src->constant_reloc();
 3444       if (rtype == relocInfo::oop_type) {
 3445         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3446       } else if (rtype == relocInfo::metadata_type) {
 3447         __ mov_metadata(dst_reg, (Metadata*)con);
 3448       } else {
 3449         assert(rtype == relocInfo::none, "unexpected reloc type");
 3450         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3451           __ mov(dst_reg, con);
 3452         } else {
 3453           uint64_t offset;
 3454           __ adrp(dst_reg, con, offset);
 3455           __ add(dst_reg, dst_reg, offset);
 3456         }
 3457       }
 3458     }
 3459   %}
 3460 
 3461   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3462     C2_MacroAssembler _masm(&cbuf);
 3463     Register dst_reg = as_Register($dst$$reg);
 3464     __ mov(dst_reg, zr);
 3465   %}
 3466 
 3467   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3468     C2_MacroAssembler _masm(&cbuf);
 3469     Register dst_reg = as_Register($dst$$reg);
 3470     __ mov(dst_reg, (uint64_t)1);
 3471   %}
 3472 
 3473   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3474     C2_MacroAssembler _masm(&cbuf);
 3475     __ load_byte_map_base($dst$$Register);
 3476   %}
 3477 
 3478   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480     Register dst_reg = as_Register($dst$$reg);
 3481     address con = (address)$src$$constant;
 3482     if (con == NULL) {
 3483       ShouldNotReachHere();
 3484     } else {
 3485       relocInfo::relocType rtype = $src->constant_reloc();
 3486       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3487       __ set_narrow_oop(dst_reg, (jobject)con);
 3488     }
 3489   %}
 3490 
 3491   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3492     C2_MacroAssembler _masm(&cbuf);
 3493     Register dst_reg = as_Register($dst$$reg);
 3494     __ mov(dst_reg, zr);
 3495   %}
 3496 
 3497   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3498     C2_MacroAssembler _masm(&cbuf);
 3499     Register dst_reg = as_Register($dst$$reg);
 3500     address con = (address)$src$$constant;
 3501     if (con == NULL) {
 3502       ShouldNotReachHere();
 3503     } else {
 3504       relocInfo::relocType rtype = $src->constant_reloc();
 3505       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3506       __ set_narrow_klass(dst_reg, (Klass *)con);
 3507     }
 3508   %}
 3509 
 3510   // arithmetic encodings
 3511 
 3512   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3513     C2_MacroAssembler _masm(&cbuf);
 3514     Register dst_reg = as_Register($dst$$reg);
 3515     Register src_reg = as_Register($src1$$reg);
 3516     int32_t con = (int32_t)$src2$$constant;
 3517     // add has primary == 0, subtract has primary == 1
 3518     if ($primary) { con = -con; }
 3519     if (con < 0) {
 3520       __ subw(dst_reg, src_reg, -con);
 3521     } else {
 3522       __ addw(dst_reg, src_reg, con);
 3523     }
 3524   %}
 3525 
 3526   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3527     C2_MacroAssembler _masm(&cbuf);
 3528     Register dst_reg = as_Register($dst$$reg);
 3529     Register src_reg = as_Register($src1$$reg);
 3530     int32_t con = (int32_t)$src2$$constant;
 3531     // add has primary == 0, subtract has primary == 1
 3532     if ($primary) { con = -con; }
 3533     if (con < 0) {
 3534       __ sub(dst_reg, src_reg, -con);
 3535     } else {
 3536       __ add(dst_reg, src_reg, con);
 3537     }
 3538   %}
 3539 
 3540   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3541     C2_MacroAssembler _masm(&cbuf);
 3542    Register dst_reg = as_Register($dst$$reg);
 3543    Register src1_reg = as_Register($src1$$reg);
 3544    Register src2_reg = as_Register($src2$$reg);
 3545     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3546   %}
 3547 
 3548   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3549     C2_MacroAssembler _masm(&cbuf);
 3550    Register dst_reg = as_Register($dst$$reg);
 3551    Register src1_reg = as_Register($src1$$reg);
 3552    Register src2_reg = as_Register($src2$$reg);
 3553     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3554   %}
 3555 
 3556   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3557     C2_MacroAssembler _masm(&cbuf);
 3558    Register dst_reg = as_Register($dst$$reg);
 3559    Register src1_reg = as_Register($src1$$reg);
 3560    Register src2_reg = as_Register($src2$$reg);
 3561     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3562   %}
 3563 
 3564   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3565     C2_MacroAssembler _masm(&cbuf);
 3566    Register dst_reg = as_Register($dst$$reg);
 3567    Register src1_reg = as_Register($src1$$reg);
 3568    Register src2_reg = as_Register($src2$$reg);
 3569     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3570   %}
 3571 
 3572   // compare instruction encodings
 3573 
 3574   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3575     C2_MacroAssembler _masm(&cbuf);
 3576     Register reg1 = as_Register($src1$$reg);
 3577     Register reg2 = as_Register($src2$$reg);
 3578     __ cmpw(reg1, reg2);
 3579   %}
 3580 
 3581   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3582     C2_MacroAssembler _masm(&cbuf);
 3583     Register reg = as_Register($src1$$reg);
 3584     int32_t val = $src2$$constant;
 3585     if (val >= 0) {
 3586       __ subsw(zr, reg, val);
 3587     } else {
 3588       __ addsw(zr, reg, -val);
 3589     }
 3590   %}
 3591 
 3592   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3593     C2_MacroAssembler _masm(&cbuf);
 3594     Register reg1 = as_Register($src1$$reg);
 3595     uint32_t val = (uint32_t)$src2$$constant;
 3596     __ movw(rscratch1, val);
 3597     __ cmpw(reg1, rscratch1);
 3598   %}
 3599 
 3600   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3601     C2_MacroAssembler _masm(&cbuf);
 3602     Register reg1 = as_Register($src1$$reg);
 3603     Register reg2 = as_Register($src2$$reg);
 3604     __ cmp(reg1, reg2);
 3605   %}
 3606 
 3607   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3608     C2_MacroAssembler _masm(&cbuf);
 3609     Register reg = as_Register($src1$$reg);
 3610     int64_t val = $src2$$constant;
 3611     if (val >= 0) {
 3612       __ subs(zr, reg, val);
 3613     } else if (val != -val) {
 3614       __ adds(zr, reg, -val);
 3615     } else {
 3616     // aargh, Long.MIN_VALUE is a special case
 3617       __ orr(rscratch1, zr, (uint64_t)val);
 3618       __ subs(zr, reg, rscratch1);
 3619     }
 3620   %}
 3621 
 3622   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3623     C2_MacroAssembler _masm(&cbuf);
 3624     Register reg1 = as_Register($src1$$reg);
 3625     uint64_t val = (uint64_t)$src2$$constant;
 3626     __ mov(rscratch1, val);
 3627     __ cmp(reg1, rscratch1);
 3628   %}
 3629 
 3630   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3631     C2_MacroAssembler _masm(&cbuf);
 3632     Register reg1 = as_Register($src1$$reg);
 3633     Register reg2 = as_Register($src2$$reg);
 3634     __ cmp(reg1, reg2);
 3635   %}
 3636 
 3637   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3638     C2_MacroAssembler _masm(&cbuf);
 3639     Register reg1 = as_Register($src1$$reg);
 3640     Register reg2 = as_Register($src2$$reg);
 3641     __ cmpw(reg1, reg2);
 3642   %}
 3643 
 3644   enc_class aarch64_enc_testp(iRegP src) %{
 3645     C2_MacroAssembler _masm(&cbuf);
 3646     Register reg = as_Register($src$$reg);
 3647     __ cmp(reg, zr);
 3648   %}
 3649 
 3650   enc_class aarch64_enc_testn(iRegN src) %{
 3651     C2_MacroAssembler _masm(&cbuf);
 3652     Register reg = as_Register($src$$reg);
 3653     __ cmpw(reg, zr);
 3654   %}
 3655 
 3656   enc_class aarch64_enc_b(label lbl) %{
 3657     C2_MacroAssembler _masm(&cbuf);
 3658     Label *L = $lbl$$label;
 3659     __ b(*L);
 3660   %}
 3661 
 3662   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3663     C2_MacroAssembler _masm(&cbuf);
 3664     Label *L = $lbl$$label;
 3665     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3666   %}
 3667 
 3668   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3669     C2_MacroAssembler _masm(&cbuf);
 3670     Label *L = $lbl$$label;
 3671     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3672   %}
 3673 
 3674   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3675   %{
 3676      Register sub_reg = as_Register($sub$$reg);
 3677      Register super_reg = as_Register($super$$reg);
 3678      Register temp_reg = as_Register($temp$$reg);
 3679      Register result_reg = as_Register($result$$reg);
 3680 
 3681      Label miss;
 3682      C2_MacroAssembler _masm(&cbuf);
 3683      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3684                                      NULL, &miss,
 3685                                      /*set_cond_codes:*/ true);
 3686      if ($primary) {
 3687        __ mov(result_reg, zr);
 3688      }
 3689      __ bind(miss);
 3690   %}
 3691 
 3692   enc_class aarch64_enc_java_static_call(method meth) %{
 3693     C2_MacroAssembler _masm(&cbuf);
 3694 
 3695     address addr = (address)$meth$$method;
 3696     address call;
 3697     if (!_method) {
 3698       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3699       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3700       if (call == NULL) {
 3701         ciEnv::current()->record_failure("CodeCache is full");
 3702         return;
 3703       }
 3704     } else {
 3705       int method_index = resolved_method_index(cbuf);
 3706       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3707                                                   : static_call_Relocation::spec(method_index);
 3708       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3709       if (call == NULL) {
 3710         ciEnv::current()->record_failure("CodeCache is full");
 3711         return;
 3712       }
 3713       // Emit stub for static call
 3714       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3715       if (stub == NULL) {
 3716         ciEnv::current()->record_failure("CodeCache is full");
 3717         return;
 3718       }
 3719     }
 3720 
 3721     __ post_call_nop();
 3722 
 3723     // Only non uncommon_trap calls need to reinitialize ptrue.
 3724     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3725       __ reinitialize_ptrue();
 3726     }
 3727   %}
 3728 
 3729   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3730     C2_MacroAssembler _masm(&cbuf);
 3731     int method_index = resolved_method_index(cbuf);
 3732     address call = __ ic_call((address)$meth$$method, method_index);
 3733     __ post_call_nop();
 3734     if (call == NULL) {
 3735       ciEnv::current()->record_failure("CodeCache is full");
 3736       return;
 3737     } else if (Compile::current()->max_vector_size() > 0) {
 3738       __ reinitialize_ptrue();
 3739     }
 3740   %}
 3741 
 3742   enc_class aarch64_enc_call_epilog() %{
 3743     C2_MacroAssembler _masm(&cbuf);
 3744     if (VerifyStackAtCalls) {
 3745       // Check that stack depth is unchanged: find majik cookie on stack
 3746       __ call_Unimplemented();
 3747     }
 3748   %}
 3749 
 3750   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3751     C2_MacroAssembler _masm(&cbuf);
 3752 
 3753     // some calls to generated routines (arraycopy code) are scheduled
 3754     // by C2 as runtime calls. if so we can call them using a br (they
 3755     // will be in a reachable segment) otherwise we have to use a blr
 3756     // which loads the absolute address into a register.
 3757     address entry = (address)$meth$$method;
 3758     CodeBlob *cb = CodeCache::find_blob(entry);
 3759     if (cb) {
 3760       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3761       __ post_call_nop();
 3762       if (call == NULL) {
 3763         ciEnv::current()->record_failure("CodeCache is full");
 3764         return;
 3765       }
 3766     } else {
 3767       Label retaddr;
 3768       __ adr(rscratch2, retaddr);
 3769       __ lea(rscratch1, RuntimeAddress(entry));
 3770       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3771       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3772       __ blr(rscratch1);
 3773       __ bind(retaddr);
 3774       // __ post_call_nop(); -- can we have one here? do we need one?
 3775       __ add(sp, sp, 2 * wordSize);
 3776     }
 3777     if (Compile::current()->max_vector_size() > 0) {
 3778       __ reinitialize_ptrue();
 3779     }
 3780   %}
 3781 
 3782   enc_class aarch64_enc_rethrow() %{
 3783     C2_MacroAssembler _masm(&cbuf);
 3784     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3785   %}
 3786 
 3787   enc_class aarch64_enc_ret() %{
 3788     C2_MacroAssembler _masm(&cbuf);
 3789 #ifdef ASSERT
 3790     if (Compile::current()->max_vector_size() > 0) {
 3791       __ verify_ptrue();
 3792     }
 3793 #endif
 3794     __ ret(lr);
 3795   %}
 3796 
 3797   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3798     C2_MacroAssembler _masm(&cbuf);
 3799     Register target_reg = as_Register($jump_target$$reg);
 3800     __ br(target_reg);
 3801   %}
 3802 
 3803   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3804     C2_MacroAssembler _masm(&cbuf);
 3805     Register target_reg = as_Register($jump_target$$reg);
 3806     // exception oop should be in r0
 3807     // ret addr has been popped into lr
 3808     // callee expects it in r3
 3809     __ mov(r3, lr);
 3810     __ br(target_reg);
 3811   %}
 3812 
 3813   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3814     C2_MacroAssembler _masm(&cbuf);
 3815     Register oop = as_Register($object$$reg);
 3816     Register box = as_Register($box$$reg);
 3817     Register disp_hdr = as_Register($tmp$$reg);
 3818     Register tmp = as_Register($tmp2$$reg);
 3819     Label cont;
 3820     Label object_has_monitor;
 3821     Label cas_failed;
 3822 
 3823     assert_different_registers(oop, box, tmp, disp_hdr);
 3824 
 3825     // Load markWord from object into displaced_header.
 3826     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3827 
 3828     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3829       __ load_klass(tmp, oop);
 3830       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3831       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3832       __ br(Assembler::NE, cont);
 3833     }
 3834 
 3835     // Check for existing monitor
 3836     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3837 
 3838     // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3839     __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3840 
 3841     // Initialize the box. (Must happen before we update the object mark!)
 3842     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3843 
 3844     // Compare object markWord with an unlocked value (tmp) and if
 3845     // equal exchange the stack address of our box with object markWord.
 3846     // On failure disp_hdr contains the possibly locked markWord.
 3847     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3848                /*release*/ true, /*weak*/ false, disp_hdr);
 3849     __ br(Assembler::EQ, cont);
 3850 
 3851     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3852 
 3853     // If the compare-and-exchange succeeded, then we found an unlocked
 3854     // object, will have now locked it will continue at label cont
 3855 
 3856     __ bind(cas_failed);
 3857     // We did not see an unlocked object so try the fast recursive case.
 3858 
 3859     // Check if the owner is self by comparing the value in the
 3860     // markWord of object (disp_hdr) with the stack pointer.
 3861     __ mov(rscratch1, sp);
 3862     __ sub(disp_hdr, disp_hdr, rscratch1);
 3863     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3864     // If condition is true we are cont and hence we can store 0 as the
 3865     // displaced header in the box, which indicates that it is a recursive lock.
 3866     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3867     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3868 
 3869     __ b(cont);
 3870 
 3871     // Handle existing monitor.
 3872     __ bind(object_has_monitor);
 3873 
 3874     // The object's monitor m is unlocked iff m->owner == NULL,
 3875     // otherwise m->owner may contain a thread or a stack address.
 3876     //
 3877     // Try to CAS m->owner from NULL to current thread.
 3878     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3879     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3880                /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
 3881 
 3882     // Store a non-null value into the box to avoid looking like a re-entrant
 3883     // lock. The fast-path monitor unlock code checks for
 3884     // markWord::monitor_value so use markWord::unused_mark which has the
 3885     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3886     __ mov(tmp, (address)markWord::unused_mark().value());
 3887     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3888 
 3889     __ bind(cont);
 3890     // flag == EQ indicates success
 3891     // flag == NE indicates failure
 3892   %}
 3893 
 3894   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3895     C2_MacroAssembler _masm(&cbuf);
 3896     Register oop = as_Register($object$$reg);
 3897     Register box = as_Register($box$$reg);
 3898     Register disp_hdr = as_Register($tmp$$reg);
 3899     Register tmp = as_Register($tmp2$$reg);
 3900     Label cont;
 3901     Label object_has_monitor;
 3902 
 3903     assert_different_registers(oop, box, tmp, disp_hdr);
 3904 
 3905     // Find the lock address and load the displaced header from the stack.
 3906     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3907 
 3908     // If the displaced header is 0, we have a recursive unlock.
 3909     __ cmp(disp_hdr, zr);
 3910     __ br(Assembler::EQ, cont);
 3911 
 3912     // Handle existing monitor.
 3913     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3914     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3915 
 3916     // Check if it is still a light weight lock, this is is true if we
 3917     // see the stack address of the basicLock in the markWord of the
 3918     // object.
 3919 
 3920     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3921                /*release*/ true, /*weak*/ false, tmp);
 3922     __ b(cont);
 3923 
 3924     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3925 
 3926     // Handle existing monitor.
 3927     __ bind(object_has_monitor);
 3928     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3929     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3930     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3931     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3932     __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
 3933     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
 3934     __ cmp(rscratch1, zr); // Sets flags for result
 3935     __ br(Assembler::NE, cont);
 3936 
 3937     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3938     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3939     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3940     __ cmp(rscratch1, zr); // Sets flags for result
 3941     __ cbnz(rscratch1, cont);
 3942     // need a release store here
 3943     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3944     __ stlr(zr, tmp); // set unowned
 3945 
 3946     __ bind(cont);
 3947     // flag == EQ indicates success
 3948     // flag == NE indicates failure
 3949   %}
 3950 
 3951 %}
 3952 
 3953 //----------FRAME--------------------------------------------------------------
 3954 // Definition of frame structure and management information.
 3955 //
 3956 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3957 //                             |   (to get allocators register number
 3958 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3959 //  r   CALLER     |        |
 3960 //  o     |        +--------+      pad to even-align allocators stack-slot
 3961 //  w     V        |  pad0  |        numbers; owned by CALLER
 3962 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3963 //  h     ^        |   in   |  5
 3964 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3965 //  |     |        |        |  3
 3966 //  |     |        +--------+
 3967 //  V     |        | old out|      Empty on Intel, window on Sparc
 3968 //        |    old |preserve|      Must be even aligned.
 3969 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3970 //        |        |   in   |  3   area for Intel ret address
 3971 //     Owned by    |preserve|      Empty on Sparc.
 3972 //       SELF      +--------+
 3973 //        |        |  pad2  |  2   pad to align old SP
 3974 //        |        +--------+  1
 3975 //        |        | locks  |  0
 3976 //        |        +--------+----> OptoReg::stack0(), even aligned
 3977 //        |        |  pad1  | 11   pad to align new SP
 3978 //        |        +--------+
 3979 //        |        |        | 10
 3980 //        |        | spills |  9   spills
 3981 //        V        |        |  8   (pad0 slot for callee)
 3982 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3983 //        ^        |  out   |  7
 3984 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3985 //     Owned by    +--------+
 3986 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3987 //        |    new |preserve|      Must be even-aligned.
 3988 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3989 //        |        |        |
 3990 //
 3991 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3992 //         known from SELF's arguments and the Java calling convention.
 3993 //         Region 6-7 is determined per call site.
 3994 // Note 2: If the calling convention leaves holes in the incoming argument
 3995 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3996 //         are owned by the CALLEE.  Holes should not be nessecary in the
 3997 //         incoming area, as the Java calling convention is completely under
 3998 //         the control of the AD file.  Doubles can be sorted and packed to
 3999 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4000 //         varargs C calling conventions.
 4001 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4002 //         even aligned with pad0 as needed.
 4003 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4004 //           (the latter is true on Intel but is it false on AArch64?)
 4005 //         region 6-11 is even aligned; it may be padded out more so that
 4006 //         the region from SP to FP meets the minimum stack alignment.
 4007 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4008 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4009 //         SP meets the minimum alignment.
 4010 
 4011 frame %{
 4012   // These three registers define part of the calling convention
 4013   // between compiled code and the interpreter.
 4014 
 4015   // Inline Cache Register or Method for I2C.
 4016   inline_cache_reg(R12);
 4017 
 4018   // Number of stack slots consumed by locking an object
 4019   sync_stack_slots(2);
 4020 
 4021   // Compiled code's Frame Pointer
 4022   frame_pointer(R31);
 4023 
 4024   // Interpreter stores its frame pointer in a register which is
 4025   // stored to the stack by I2CAdaptors.
 4026   // I2CAdaptors convert from interpreted java to compiled java.
 4027   interpreter_frame_pointer(R29);
 4028 
 4029   // Stack alignment requirement
 4030   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4031 
 4032   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4033   // for calls to C.  Supports the var-args backing area for register parms.
 4034   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4035 
 4036   // The after-PROLOG location of the return address.  Location of
 4037   // return address specifies a type (REG or STACK) and a number
 4038   // representing the register number (i.e. - use a register name) or
 4039   // stack slot.
 4040   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4041   // Otherwise, it is above the locks and verification slot and alignment word
 4042   // TODO this may well be correct but need to check why that - 2 is there
 4043   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4044   // which folds in the space used for monitors
 4045   return_addr(STACK - 2 +
 4046               align_up((Compile::current()->in_preserve_stack_slots() +
 4047                         Compile::current()->fixed_slots()),
 4048                        stack_alignment_in_slots()));
 4049 
 4050   // Location of compiled Java return values.  Same as C for now.
 4051   return_value
 4052   %{
 4053     // TODO do we allow ideal_reg == Op_RegN???
 4054     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4055            "only return normal values");
 4056 
 4057     static const int lo[Op_RegL + 1] = { // enum name
 4058       0,                                 // Op_Node
 4059       0,                                 // Op_Set
 4060       R0_num,                            // Op_RegN
 4061       R0_num,                            // Op_RegI
 4062       R0_num,                            // Op_RegP
 4063       V0_num,                            // Op_RegF
 4064       V0_num,                            // Op_RegD
 4065       R0_num                             // Op_RegL
 4066     };
 4067 
 4068     static const int hi[Op_RegL + 1] = { // enum name
 4069       0,                                 // Op_Node
 4070       0,                                 // Op_Set
 4071       OptoReg::Bad,                      // Op_RegN
 4072       OptoReg::Bad,                      // Op_RegI
 4073       R0_H_num,                          // Op_RegP
 4074       OptoReg::Bad,                      // Op_RegF
 4075       V0_H_num,                          // Op_RegD
 4076       R0_H_num                           // Op_RegL
 4077     };
 4078 
 4079     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4080   %}
 4081 %}
 4082 
 4083 //----------ATTRIBUTES---------------------------------------------------------
 4084 //----------Operand Attributes-------------------------------------------------
 4085 op_attrib op_cost(1);        // Required cost attribute
 4086 
 4087 //----------Instruction Attributes---------------------------------------------
 4088 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4089 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4090 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4091                                 // a non-matching short branch variant
 4092                                 // of some long branch?
 4093 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4094                                 // be a power of 2) specifies the
 4095                                 // alignment that some part of the
 4096                                 // instruction (not necessarily the
 4097                                 // start) requires.  If > 1, a
 4098                                 // compute_padding() function must be
 4099                                 // provided for the instruction
 4100 
 4101 //----------OPERANDS-----------------------------------------------------------
 4102 // Operand definitions must precede instruction definitions for correct parsing
 4103 // in the ADLC because operands constitute user defined types which are used in
 4104 // instruction definitions.
 4105 
 4106 //----------Simple Operands----------------------------------------------------
 4107 
 4108 // Integer operands 32 bit
 4109 // 32 bit immediate
 4110 operand immI()
 4111 %{
 4112   match(ConI);
 4113 
 4114   op_cost(0);
 4115   format %{ %}
 4116   interface(CONST_INTER);
 4117 %}
 4118 
 4119 // 32 bit zero
 4120 operand immI0()
 4121 %{
 4122   predicate(n->get_int() == 0);
 4123   match(ConI);
 4124 
 4125   op_cost(0);
 4126   format %{ %}
 4127   interface(CONST_INTER);
 4128 %}
 4129 
 4130 // 32 bit unit increment
 4131 operand immI_1()
 4132 %{
 4133   predicate(n->get_int() == 1);
 4134   match(ConI);
 4135 
 4136   op_cost(0);
 4137   format %{ %}
 4138   interface(CONST_INTER);
 4139 %}
 4140 
 4141 // 32 bit unit decrement
 4142 operand immI_M1()
 4143 %{
 4144   predicate(n->get_int() == -1);
 4145   match(ConI);
 4146 
 4147   op_cost(0);
 4148   format %{ %}
 4149   interface(CONST_INTER);
 4150 %}
 4151 
 4152 // Shift values for add/sub extension shift
 4153 operand immIExt()
 4154 %{
 4155   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4156   match(ConI);
 4157 
 4158   op_cost(0);
 4159   format %{ %}
 4160   interface(CONST_INTER);
 4161 %}
 4162 
 4163 operand immI_gt_1()
 4164 %{
 4165   predicate(n->get_int() > 1);
 4166   match(ConI);
 4167 
 4168   op_cost(0);
 4169   format %{ %}
 4170   interface(CONST_INTER);
 4171 %}
 4172 
 4173 operand immI_le_4()
 4174 %{
 4175   predicate(n->get_int() <= 4);
 4176   match(ConI);
 4177 
 4178   op_cost(0);
 4179   format %{ %}
 4180   interface(CONST_INTER);
 4181 %}
 4182 
 4183 operand immI_31()
 4184 %{
 4185   predicate(n->get_int() == 31);
 4186   match(ConI);
 4187 
 4188   op_cost(0);
 4189   format %{ %}
 4190   interface(CONST_INTER);
 4191 %}
 4192 
 4193 operand immI_2()
 4194 %{
 4195   predicate(n->get_int() == 2);
 4196   match(ConI);
 4197 
 4198   op_cost(0);
 4199   format %{ %}
 4200   interface(CONST_INTER);
 4201 %}
 4202 
 4203 operand immI_4()
 4204 %{
 4205   predicate(n->get_int() == 4);
 4206   match(ConI);
 4207 
 4208   op_cost(0);
 4209   format %{ %}
 4210   interface(CONST_INTER);
 4211 %}
 4212 
 4213 operand immI_8()
 4214 %{
 4215   predicate(n->get_int() == 8);
 4216   match(ConI);
 4217 
 4218   op_cost(0);
 4219   format %{ %}
 4220   interface(CONST_INTER);
 4221 %}
 4222 
 4223 operand immI_16()
 4224 %{
 4225   predicate(n->get_int() == 16);
 4226   match(ConI);
 4227 
 4228   op_cost(0);
 4229   format %{ %}
 4230   interface(CONST_INTER);
 4231 %}
 4232 
 4233 operand immI_24()
 4234 %{
 4235   predicate(n->get_int() == 24);
 4236   match(ConI);
 4237 
 4238   op_cost(0);
 4239   format %{ %}
 4240   interface(CONST_INTER);
 4241 %}
 4242 
 4243 operand immI_32()
 4244 %{
 4245   predicate(n->get_int() == 32);
 4246   match(ConI);
 4247 
 4248   op_cost(0);
 4249   format %{ %}
 4250   interface(CONST_INTER);
 4251 %}
 4252 
 4253 operand immI_48()
 4254 %{
 4255   predicate(n->get_int() == 48);
 4256   match(ConI);
 4257 
 4258   op_cost(0);
 4259   format %{ %}
 4260   interface(CONST_INTER);
 4261 %}
 4262 
 4263 operand immI_56()
 4264 %{
 4265   predicate(n->get_int() == 56);
 4266   match(ConI);
 4267 
 4268   op_cost(0);
 4269   format %{ %}
 4270   interface(CONST_INTER);
 4271 %}
 4272 
 4273 operand immI_63()
 4274 %{
 4275   predicate(n->get_int() == 63);
 4276   match(ConI);
 4277 
 4278   op_cost(0);
 4279   format %{ %}
 4280   interface(CONST_INTER);
 4281 %}
 4282 
 4283 operand immI_64()
 4284 %{
 4285   predicate(n->get_int() == 64);
 4286   match(ConI);
 4287 
 4288   op_cost(0);
 4289   format %{ %}
 4290   interface(CONST_INTER);
 4291 %}
 4292 
 4293 operand immI_255()
 4294 %{
 4295   predicate(n->get_int() == 255);
 4296   match(ConI);
 4297 
 4298   op_cost(0);
 4299   format %{ %}
 4300   interface(CONST_INTER);
 4301 %}
 4302 
 4303 operand immI_65535()
 4304 %{
 4305   predicate(n->get_int() == 65535);
 4306   match(ConI);
 4307 
 4308   op_cost(0);
 4309   format %{ %}
 4310   interface(CONST_INTER);
 4311 %}
 4312 
 4313 operand immL_255()
 4314 %{
 4315   predicate(n->get_long() == 255L);
 4316   match(ConL);
 4317 
 4318   op_cost(0);
 4319   format %{ %}
 4320   interface(CONST_INTER);
 4321 %}
 4322 
 4323 operand immL_65535()
 4324 %{
 4325   predicate(n->get_long() == 65535L);
 4326   match(ConL);
 4327 
 4328   op_cost(0);
 4329   format %{ %}
 4330   interface(CONST_INTER);
 4331 %}
 4332 
 4333 operand immL_4294967295()
 4334 %{
 4335   predicate(n->get_long() == 4294967295L);
 4336   match(ConL);
 4337 
 4338   op_cost(0);
 4339   format %{ %}
 4340   interface(CONST_INTER);
 4341 %}
 4342 
 4343 operand immL_bitmask()
 4344 %{
 4345   predicate((n->get_long() != 0)
 4346             && ((n->get_long() & 0xc000000000000000l) == 0)
 4347             && is_power_of_2(n->get_long() + 1));
 4348   match(ConL);
 4349 
 4350   op_cost(0);
 4351   format %{ %}
 4352   interface(CONST_INTER);
 4353 %}
 4354 
 4355 operand immI_bitmask()
 4356 %{
 4357   predicate((n->get_int() != 0)
 4358             && ((n->get_int() & 0xc0000000) == 0)
 4359             && is_power_of_2(n->get_int() + 1));
 4360   match(ConI);
 4361 
 4362   op_cost(0);
 4363   format %{ %}
 4364   interface(CONST_INTER);
 4365 %}
 4366 
 4367 operand immL_positive_bitmaskI()
 4368 %{
 4369   predicate((n->get_long() != 0)
 4370             && ((julong)n->get_long() < 0x80000000ULL)
 4371             && is_power_of_2(n->get_long() + 1));
 4372   match(ConL);
 4373 
 4374   op_cost(0);
 4375   format %{ %}
 4376   interface(CONST_INTER);
 4377 %}
 4378 
 4379 // Scale values for scaled offset addressing modes (up to long but not quad)
 4380 operand immIScale()
 4381 %{
 4382   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4383   match(ConI);
 4384 
 4385   op_cost(0);
 4386   format %{ %}
 4387   interface(CONST_INTER);
 4388 %}
 4389 
 4390 // 26 bit signed offset -- for pc-relative branches
 4391 operand immI26()
 4392 %{
 4393   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4394   match(ConI);
 4395 
 4396   op_cost(0);
 4397   format %{ %}
 4398   interface(CONST_INTER);
 4399 %}
 4400 
 4401 // 19 bit signed offset -- for pc-relative loads
 4402 operand immI19()
 4403 %{
 4404   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4405   match(ConI);
 4406 
 4407   op_cost(0);
 4408   format %{ %}
 4409   interface(CONST_INTER);
 4410 %}
 4411 
 4412 // 12 bit unsigned offset -- for base plus immediate loads
 4413 operand immIU12()
 4414 %{
 4415   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4416   match(ConI);
 4417 
 4418   op_cost(0);
 4419   format %{ %}
 4420   interface(CONST_INTER);
 4421 %}
 4422 
 4423 operand immLU12()
 4424 %{
 4425   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4426   match(ConL);
 4427 
 4428   op_cost(0);
 4429   format %{ %}
 4430   interface(CONST_INTER);
 4431 %}
 4432 
 4433 // Offset for scaled or unscaled immediate loads and stores
 4434 operand immIOffset()
 4435 %{
 4436   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4437   match(ConI);
 4438 
 4439   op_cost(0);
 4440   format %{ %}
 4441   interface(CONST_INTER);
 4442 %}
 4443 
 4444 operand immIOffset1()
 4445 %{
 4446   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4447   match(ConI);
 4448 
 4449   op_cost(0);
 4450   format %{ %}
 4451   interface(CONST_INTER);
 4452 %}
 4453 
 4454 operand immIOffset2()
 4455 %{
 4456   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4457   match(ConI);
 4458 
 4459   op_cost(0);
 4460   format %{ %}
 4461   interface(CONST_INTER);
 4462 %}
 4463 
 4464 operand immIOffset4()
 4465 %{
 4466   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4467   match(ConI);
 4468 
 4469   op_cost(0);
 4470   format %{ %}
 4471   interface(CONST_INTER);
 4472 %}
 4473 
 4474 operand immIOffset8()
 4475 %{
 4476   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4477   match(ConI);
 4478 
 4479   op_cost(0);
 4480   format %{ %}
 4481   interface(CONST_INTER);
 4482 %}
 4483 
 4484 operand immIOffset16()
 4485 %{
 4486   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4487   match(ConI);
 4488 
 4489   op_cost(0);
 4490   format %{ %}
 4491   interface(CONST_INTER);
 4492 %}
 4493 
 4494 operand immLoffset()
 4495 %{
 4496   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4497   match(ConL);
 4498 
 4499   op_cost(0);
 4500   format %{ %}
 4501   interface(CONST_INTER);
 4502 %}
 4503 
 4504 operand immLoffset1()
 4505 %{
 4506   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4507   match(ConL);
 4508 
 4509   op_cost(0);
 4510   format %{ %}
 4511   interface(CONST_INTER);
 4512 %}
 4513 
 4514 operand immLoffset2()
 4515 %{
 4516   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4517   match(ConL);
 4518 
 4519   op_cost(0);
 4520   format %{ %}
 4521   interface(CONST_INTER);
 4522 %}
 4523 
 4524 operand immLoffset4()
 4525 %{
 4526   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4527   match(ConL);
 4528 
 4529   op_cost(0);
 4530   format %{ %}
 4531   interface(CONST_INTER);
 4532 %}
 4533 
 4534 operand immLoffset8()
 4535 %{
 4536   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4537   match(ConL);
 4538 
 4539   op_cost(0);
 4540   format %{ %}
 4541   interface(CONST_INTER);
 4542 %}
 4543 
 4544 operand immLoffset16()
 4545 %{
 4546   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4547   match(ConL);
 4548 
 4549   op_cost(0);
 4550   format %{ %}
 4551   interface(CONST_INTER);
 4552 %}
 4553 
 4554 // 8 bit signed value.
 4555 operand immI8()
 4556 %{
 4557   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4558   match(ConI);
 4559 
 4560   op_cost(0);
 4561   format %{ %}
 4562   interface(CONST_INTER);
 4563 %}
 4564 
 4565 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4566 operand immI8_shift8()
 4567 %{
 4568   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4569             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4570   match(ConI);
 4571 
 4572   op_cost(0);
 4573   format %{ %}
 4574   interface(CONST_INTER);
 4575 %}
 4576 
 4577 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4578 operand immL8_shift8()
 4579 %{
 4580   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4581             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4582   match(ConL);
 4583 
 4584   op_cost(0);
 4585   format %{ %}
 4586   interface(CONST_INTER);
 4587 %}
 4588 
 4589 // 32 bit integer valid for add sub immediate
 4590 operand immIAddSub()
 4591 %{
 4592   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4593   match(ConI);
 4594   op_cost(0);
 4595   format %{ %}
 4596   interface(CONST_INTER);
 4597 %}
 4598 
 4599 // 32 bit unsigned integer valid for logical immediate
 4600 // TODO -- check this is right when e.g the mask is 0x80000000
 4601 operand immILog()
 4602 %{
 4603   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4604   match(ConI);
 4605 
 4606   op_cost(0);
 4607   format %{ %}
 4608   interface(CONST_INTER);
 4609 %}
 4610 
 4611 // Integer operands 64 bit
 4612 // 64 bit immediate
 4613 operand immL()
 4614 %{
 4615   match(ConL);
 4616 
 4617   op_cost(0);
 4618   format %{ %}
 4619   interface(CONST_INTER);
 4620 %}
 4621 
 4622 // 64 bit zero
 4623 operand immL0()
 4624 %{
 4625   predicate(n->get_long() == 0);
 4626   match(ConL);
 4627 
 4628   op_cost(0);
 4629   format %{ %}
 4630   interface(CONST_INTER);
 4631 %}
 4632 
 4633 // 64 bit unit increment
 4634 operand immL_1()
 4635 %{
 4636   predicate(n->get_long() == 1);
 4637   match(ConL);
 4638 
 4639   op_cost(0);
 4640   format %{ %}
 4641   interface(CONST_INTER);
 4642 %}
 4643 
 4644 // 64 bit unit decrement
 4645 operand immL_M1()
 4646 %{
 4647   predicate(n->get_long() == -1);
 4648   match(ConL);
 4649 
 4650   op_cost(0);
 4651   format %{ %}
 4652   interface(CONST_INTER);
 4653 %}
 4654 
 4655 // 32 bit offset of pc in thread anchor
 4656 
 4657 operand immL_pc_off()
 4658 %{
 4659   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4660                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4661   match(ConL);
 4662 
 4663   op_cost(0);
 4664   format %{ %}
 4665   interface(CONST_INTER);
 4666 %}
 4667 
 4668 // 64 bit integer valid for add sub immediate
 4669 operand immLAddSub()
 4670 %{
 4671   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4672   match(ConL);
 4673   op_cost(0);
 4674   format %{ %}
 4675   interface(CONST_INTER);
 4676 %}
 4677 
 4678 // 64 bit integer valid for logical immediate
 4679 operand immLLog()
 4680 %{
 4681   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4682   match(ConL);
 4683   op_cost(0);
 4684   format %{ %}
 4685   interface(CONST_INTER);
 4686 %}
 4687 
 4688 // Long Immediate: low 32-bit mask
 4689 operand immL_32bits()
 4690 %{
 4691   predicate(n->get_long() == 0xFFFFFFFFL);
 4692   match(ConL);
 4693   op_cost(0);
 4694   format %{ %}
 4695   interface(CONST_INTER);
 4696 %}
 4697 
 4698 // Pointer operands
 4699 // Pointer Immediate
 4700 operand immP()
 4701 %{
 4702   match(ConP);
 4703 
 4704   op_cost(0);
 4705   format %{ %}
 4706   interface(CONST_INTER);
 4707 %}
 4708 
 4709 // NULL Pointer Immediate
 4710 operand immP0()
 4711 %{
 4712   predicate(n->get_ptr() == 0);
 4713   match(ConP);
 4714 
 4715   op_cost(0);
 4716   format %{ %}
 4717   interface(CONST_INTER);
 4718 %}
 4719 
 4720 // Pointer Immediate One
 4721 // this is used in object initialization (initial object header)
 4722 operand immP_1()
 4723 %{
 4724   predicate(n->get_ptr() == 1);
 4725   match(ConP);
 4726 
 4727   op_cost(0);
 4728   format %{ %}
 4729   interface(CONST_INTER);
 4730 %}
 4731 
 4732 // Card Table Byte Map Base
 4733 operand immByteMapBase()
 4734 %{
 4735   // Get base of card map
 4736   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4737             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4738   match(ConP);
 4739 
 4740   op_cost(0);
 4741   format %{ %}
 4742   interface(CONST_INTER);
 4743 %}
 4744 
 4745 // Pointer Immediate Minus One
 4746 // this is used when we want to write the current PC to the thread anchor
 4747 operand immP_M1()
 4748 %{
 4749   predicate(n->get_ptr() == -1);
 4750   match(ConP);
 4751 
 4752   op_cost(0);
 4753   format %{ %}
 4754   interface(CONST_INTER);
 4755 %}
 4756 
 4757 // Pointer Immediate Minus Two
 4758 // this is used when we want to write the current PC to the thread anchor
 4759 operand immP_M2()
 4760 %{
 4761   predicate(n->get_ptr() == -2);
 4762   match(ConP);
 4763 
 4764   op_cost(0);
 4765   format %{ %}
 4766   interface(CONST_INTER);
 4767 %}
 4768 
 4769 // Float and Double operands
 4770 // Double Immediate
 4771 operand immD()
 4772 %{
 4773   match(ConD);
 4774   op_cost(0);
 4775   format %{ %}
 4776   interface(CONST_INTER);
 4777 %}
 4778 
 4779 // Double Immediate: +0.0d
 4780 operand immD0()
 4781 %{
 4782   predicate(jlong_cast(n->getd()) == 0);
 4783   match(ConD);
 4784 
 4785   op_cost(0);
 4786   format %{ %}
 4787   interface(CONST_INTER);
 4788 %}
 4789 
 4790 // constant 'double +0.0'.
 4791 operand immDPacked()
 4792 %{
 4793   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4794   match(ConD);
 4795   op_cost(0);
 4796   format %{ %}
 4797   interface(CONST_INTER);
 4798 %}
 4799 
 4800 // Float Immediate
 4801 operand immF()
 4802 %{
 4803   match(ConF);
 4804   op_cost(0);
 4805   format %{ %}
 4806   interface(CONST_INTER);
 4807 %}
 4808 
 4809 // Float Immediate: +0.0f.
 4810 operand immF0()
 4811 %{
 4812   predicate(jint_cast(n->getf()) == 0);
 4813   match(ConF);
 4814 
 4815   op_cost(0);
 4816   format %{ %}
 4817   interface(CONST_INTER);
 4818 %}
 4819 
 4820 //
 4821 operand immFPacked()
 4822 %{
 4823   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4824   match(ConF);
 4825   op_cost(0);
 4826   format %{ %}
 4827   interface(CONST_INTER);
 4828 %}
 4829 
 4830 // Narrow pointer operands
 4831 // Narrow Pointer Immediate
 4832 operand immN()
 4833 %{
 4834   match(ConN);
 4835 
 4836   op_cost(0);
 4837   format %{ %}
 4838   interface(CONST_INTER);
 4839 %}
 4840 
 4841 // Narrow NULL Pointer Immediate
 4842 operand immN0()
 4843 %{
 4844   predicate(n->get_narrowcon() == 0);
 4845   match(ConN);
 4846 
 4847   op_cost(0);
 4848   format %{ %}
 4849   interface(CONST_INTER);
 4850 %}
 4851 
 4852 operand immNKlass()
 4853 %{
 4854   match(ConNKlass);
 4855 
 4856   op_cost(0);
 4857   format %{ %}
 4858   interface(CONST_INTER);
 4859 %}
 4860 
 4861 // Integer 32 bit Register Operands
 4862 // Integer 32 bitRegister (excludes SP)
 4863 operand iRegI()
 4864 %{
 4865   constraint(ALLOC_IN_RC(any_reg32));
 4866   match(RegI);
 4867   match(iRegINoSp);
 4868   op_cost(0);
 4869   format %{ %}
 4870   interface(REG_INTER);
 4871 %}
 4872 
 4873 // Integer 32 bit Register not Special
 4874 operand iRegINoSp()
 4875 %{
 4876   constraint(ALLOC_IN_RC(no_special_reg32));
 4877   match(RegI);
 4878   op_cost(0);
 4879   format %{ %}
 4880   interface(REG_INTER);
 4881 %}
 4882 
 4883 // Integer 64 bit Register Operands
 4884 // Integer 64 bit Register (includes SP)
 4885 operand iRegL()
 4886 %{
 4887   constraint(ALLOC_IN_RC(any_reg));
 4888   match(RegL);
 4889   match(iRegLNoSp);
 4890   op_cost(0);
 4891   format %{ %}
 4892   interface(REG_INTER);
 4893 %}
 4894 
 4895 // Integer 64 bit Register not Special
 4896 operand iRegLNoSp()
 4897 %{
 4898   constraint(ALLOC_IN_RC(no_special_reg));
 4899   match(RegL);
 4900   match(iRegL_R0);
 4901   format %{ %}
 4902   interface(REG_INTER);
 4903 %}
 4904 
 4905 // Pointer Register Operands
 4906 // Pointer Register
 4907 operand iRegP()
 4908 %{
 4909   constraint(ALLOC_IN_RC(ptr_reg));
 4910   match(RegP);
 4911   match(iRegPNoSp);
 4912   match(iRegP_R0);
 4913   //match(iRegP_R2);
 4914   //match(iRegP_R4);
 4915   //match(iRegP_R5);
 4916   match(thread_RegP);
 4917   op_cost(0);
 4918   format %{ %}
 4919   interface(REG_INTER);
 4920 %}
 4921 
 4922 // Pointer 64 bit Register not Special
 4923 operand iRegPNoSp()
 4924 %{
 4925   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4926   match(RegP);
 4927   // match(iRegP);
 4928   // match(iRegP_R0);
 4929   // match(iRegP_R2);
 4930   // match(iRegP_R4);
 4931   // match(iRegP_R5);
 4932   // match(thread_RegP);
 4933   op_cost(0);
 4934   format %{ %}
 4935   interface(REG_INTER);
 4936 %}
 4937 
 4938 // Pointer 64 bit Register R0 only
 4939 operand iRegP_R0()
 4940 %{
 4941   constraint(ALLOC_IN_RC(r0_reg));
 4942   match(RegP);
 4943   // match(iRegP);
 4944   match(iRegPNoSp);
 4945   op_cost(0);
 4946   format %{ %}
 4947   interface(REG_INTER);
 4948 %}
 4949 
 4950 // Pointer 64 bit Register R1 only
 4951 operand iRegP_R1()
 4952 %{
 4953   constraint(ALLOC_IN_RC(r1_reg));
 4954   match(RegP);
 4955   // match(iRegP);
 4956   match(iRegPNoSp);
 4957   op_cost(0);
 4958   format %{ %}
 4959   interface(REG_INTER);
 4960 %}
 4961 
 4962 // Pointer 64 bit Register R2 only
 4963 operand iRegP_R2()
 4964 %{
 4965   constraint(ALLOC_IN_RC(r2_reg));
 4966   match(RegP);
 4967   // match(iRegP);
 4968   match(iRegPNoSp);
 4969   op_cost(0);
 4970   format %{ %}
 4971   interface(REG_INTER);
 4972 %}
 4973 
 4974 // Pointer 64 bit Register R3 only
 4975 operand iRegP_R3()
 4976 %{
 4977   constraint(ALLOC_IN_RC(r3_reg));
 4978   match(RegP);
 4979   // match(iRegP);
 4980   match(iRegPNoSp);
 4981   op_cost(0);
 4982   format %{ %}
 4983   interface(REG_INTER);
 4984 %}
 4985 
 4986 // Pointer 64 bit Register R4 only
 4987 operand iRegP_R4()
 4988 %{
 4989   constraint(ALLOC_IN_RC(r4_reg));
 4990   match(RegP);
 4991   // match(iRegP);
 4992   match(iRegPNoSp);
 4993   op_cost(0);
 4994   format %{ %}
 4995   interface(REG_INTER);
 4996 %}
 4997 
 4998 // Pointer 64 bit Register R5 only
 4999 operand iRegP_R5()
 5000 %{
 5001   constraint(ALLOC_IN_RC(r5_reg));
 5002   match(RegP);
 5003   // match(iRegP);
 5004   match(iRegPNoSp);
 5005   op_cost(0);
 5006   format %{ %}
 5007   interface(REG_INTER);
 5008 %}
 5009 
 5010 // Pointer 64 bit Register R10 only
 5011 operand iRegP_R10()
 5012 %{
 5013   constraint(ALLOC_IN_RC(r10_reg));
 5014   match(RegP);
 5015   // match(iRegP);
 5016   match(iRegPNoSp);
 5017   op_cost(0);
 5018   format %{ %}
 5019   interface(REG_INTER);
 5020 %}
 5021 
 5022 // Long 64 bit Register R0 only
 5023 operand iRegL_R0()
 5024 %{
 5025   constraint(ALLOC_IN_RC(r0_reg));
 5026   match(RegL);
 5027   match(iRegLNoSp);
 5028   op_cost(0);
 5029   format %{ %}
 5030   interface(REG_INTER);
 5031 %}
 5032 
 5033 // Long 64 bit Register R2 only
 5034 operand iRegL_R2()
 5035 %{
 5036   constraint(ALLOC_IN_RC(r2_reg));
 5037   match(RegL);
 5038   match(iRegLNoSp);
 5039   op_cost(0);
 5040   format %{ %}
 5041   interface(REG_INTER);
 5042 %}
 5043 
 5044 // Long 64 bit Register R3 only
 5045 operand iRegL_R3()
 5046 %{
 5047   constraint(ALLOC_IN_RC(r3_reg));
 5048   match(RegL);
 5049   match(iRegLNoSp);
 5050   op_cost(0);
 5051   format %{ %}
 5052   interface(REG_INTER);
 5053 %}
 5054 
 5055 // Long 64 bit Register R11 only
 5056 operand iRegL_R11()
 5057 %{
 5058   constraint(ALLOC_IN_RC(r11_reg));
 5059   match(RegL);
 5060   match(iRegLNoSp);
 5061   op_cost(0);
 5062   format %{ %}
 5063   interface(REG_INTER);
 5064 %}
 5065 
 5066 // Pointer 64 bit Register FP only
 5067 operand iRegP_FP()
 5068 %{
 5069   constraint(ALLOC_IN_RC(fp_reg));
 5070   match(RegP);
 5071   // match(iRegP);
 5072   op_cost(0);
 5073   format %{ %}
 5074   interface(REG_INTER);
 5075 %}
 5076 
 5077 // Register R0 only
 5078 operand iRegI_R0()
 5079 %{
 5080   constraint(ALLOC_IN_RC(int_r0_reg));
 5081   match(RegI);
 5082   match(iRegINoSp);
 5083   op_cost(0);
 5084   format %{ %}
 5085   interface(REG_INTER);
 5086 %}
 5087 
 5088 // Register R2 only
 5089 operand iRegI_R2()
 5090 %{
 5091   constraint(ALLOC_IN_RC(int_r2_reg));
 5092   match(RegI);
 5093   match(iRegINoSp);
 5094   op_cost(0);
 5095   format %{ %}
 5096   interface(REG_INTER);
 5097 %}
 5098 
 5099 // Register R3 only
 5100 operand iRegI_R3()
 5101 %{
 5102   constraint(ALLOC_IN_RC(int_r3_reg));
 5103   match(RegI);
 5104   match(iRegINoSp);
 5105   op_cost(0);
 5106   format %{ %}
 5107   interface(REG_INTER);
 5108 %}
 5109 
 5110 
 5111 // Register R4 only
 5112 operand iRegI_R4()
 5113 %{
 5114   constraint(ALLOC_IN_RC(int_r4_reg));
 5115   match(RegI);
 5116   match(iRegINoSp);
 5117   op_cost(0);
 5118   format %{ %}
 5119   interface(REG_INTER);
 5120 %}
 5121 
 5122 
 5123 // Pointer Register Operands
 5124 // Narrow Pointer Register
 5125 operand iRegN()
 5126 %{
 5127   constraint(ALLOC_IN_RC(any_reg32));
 5128   match(RegN);
 5129   match(iRegNNoSp);
 5130   op_cost(0);
 5131   format %{ %}
 5132   interface(REG_INTER);
 5133 %}
 5134 
 5135 operand iRegN_R0()
 5136 %{
 5137   constraint(ALLOC_IN_RC(r0_reg));
 5138   match(iRegN);
 5139   op_cost(0);
 5140   format %{ %}
 5141   interface(REG_INTER);
 5142 %}
 5143 
 5144 operand iRegN_R2()
 5145 %{
 5146   constraint(ALLOC_IN_RC(r2_reg));
 5147   match(iRegN);
 5148   op_cost(0);
 5149   format %{ %}
 5150   interface(REG_INTER);
 5151 %}
 5152 
 5153 operand iRegN_R3()
 5154 %{
 5155   constraint(ALLOC_IN_RC(r3_reg));
 5156   match(iRegN);
 5157   op_cost(0);
 5158   format %{ %}
 5159   interface(REG_INTER);
 5160 %}
 5161 
 5162 // Integer 64 bit Register not Special
 5163 operand iRegNNoSp()
 5164 %{
 5165   constraint(ALLOC_IN_RC(no_special_reg32));
 5166   match(RegN);
 5167   op_cost(0);
 5168   format %{ %}
 5169   interface(REG_INTER);
 5170 %}
 5171 
 5172 // heap base register -- used for encoding immN0
 5173 
 5174 operand iRegIHeapbase()
 5175 %{
 5176   constraint(ALLOC_IN_RC(heapbase_reg));
 5177   match(RegI);
 5178   op_cost(0);
 5179   format %{ %}
 5180   interface(REG_INTER);
 5181 %}
 5182 
 5183 // Float Register
 5184 // Float register operands
 5185 operand vRegF()
 5186 %{
 5187   constraint(ALLOC_IN_RC(float_reg));
 5188   match(RegF);
 5189 
 5190   op_cost(0);
 5191   format %{ %}
 5192   interface(REG_INTER);
 5193 %}
 5194 
 5195 // Double Register
 5196 // Double register operands
 5197 operand vRegD()
 5198 %{
 5199   constraint(ALLOC_IN_RC(double_reg));
 5200   match(RegD);
 5201 
 5202   op_cost(0);
 5203   format %{ %}
 5204   interface(REG_INTER);
 5205 %}
 5206 
 5207 // Generic vector class. This will be used for
 5208 // all vector operands, including NEON and SVE,
 5209 // but currently only used for SVE VecA.
 5210 operand vReg()
 5211 %{
 5212   constraint(ALLOC_IN_RC(vectora_reg));
 5213   match(VecA);
 5214   op_cost(0);
 5215   format %{ %}
 5216   interface(REG_INTER);
 5217 %}
 5218 
 5219 operand vecD()
 5220 %{
 5221   constraint(ALLOC_IN_RC(vectord_reg));
 5222   match(VecD);
 5223 
 5224   op_cost(0);
 5225   format %{ %}
 5226   interface(REG_INTER);
 5227 %}
 5228 
 5229 operand vecX()
 5230 %{
 5231   constraint(ALLOC_IN_RC(vectorx_reg));
 5232   match(VecX);
 5233 
 5234   op_cost(0);
 5235   format %{ %}
 5236   interface(REG_INTER);
 5237 %}
 5238 
 5239 operand vRegD_V0()
 5240 %{
 5241   constraint(ALLOC_IN_RC(v0_reg));
 5242   match(RegD);
 5243   op_cost(0);
 5244   format %{ %}
 5245   interface(REG_INTER);
 5246 %}
 5247 
 5248 operand vRegD_V1()
 5249 %{
 5250   constraint(ALLOC_IN_RC(v1_reg));
 5251   match(RegD);
 5252   op_cost(0);
 5253   format %{ %}
 5254   interface(REG_INTER);
 5255 %}
 5256 
 5257 operand vRegD_V2()
 5258 %{
 5259   constraint(ALLOC_IN_RC(v2_reg));
 5260   match(RegD);
 5261   op_cost(0);
 5262   format %{ %}
 5263   interface(REG_INTER);
 5264 %}
 5265 
 5266 operand vRegD_V3()
 5267 %{
 5268   constraint(ALLOC_IN_RC(v3_reg));
 5269   match(RegD);
 5270   op_cost(0);
 5271   format %{ %}
 5272   interface(REG_INTER);
 5273 %}
 5274 
 5275 operand vRegD_V4()
 5276 %{
 5277   constraint(ALLOC_IN_RC(v4_reg));
 5278   match(RegD);
 5279   op_cost(0);
 5280   format %{ %}
 5281   interface(REG_INTER);
 5282 %}
 5283 
 5284 operand vRegD_V5()
 5285 %{
 5286   constraint(ALLOC_IN_RC(v5_reg));
 5287   match(RegD);
 5288   op_cost(0);
 5289   format %{ %}
 5290   interface(REG_INTER);
 5291 %}
 5292 
 5293 operand vRegD_V6()
 5294 %{
 5295   constraint(ALLOC_IN_RC(v6_reg));
 5296   match(RegD);
 5297   op_cost(0);
 5298   format %{ %}
 5299   interface(REG_INTER);
 5300 %}
 5301 
 5302 operand vRegD_V7()
 5303 %{
 5304   constraint(ALLOC_IN_RC(v7_reg));
 5305   match(RegD);
 5306   op_cost(0);
 5307   format %{ %}
 5308   interface(REG_INTER);
 5309 %}
 5310 
 5311 operand vRegD_V8()
 5312 %{
 5313   constraint(ALLOC_IN_RC(v8_reg));
 5314   match(RegD);
 5315   op_cost(0);
 5316   format %{ %}
 5317   interface(REG_INTER);
 5318 %}
 5319 
 5320 operand vRegD_V9()
 5321 %{
 5322   constraint(ALLOC_IN_RC(v9_reg));
 5323   match(RegD);
 5324   op_cost(0);
 5325   format %{ %}
 5326   interface(REG_INTER);
 5327 %}
 5328 
 5329 operand vRegD_V10()
 5330 %{
 5331   constraint(ALLOC_IN_RC(v10_reg));
 5332   match(RegD);
 5333   op_cost(0);
 5334   format %{ %}
 5335   interface(REG_INTER);
 5336 %}
 5337 
 5338 operand vRegD_V11()
 5339 %{
 5340   constraint(ALLOC_IN_RC(v11_reg));
 5341   match(RegD);
 5342   op_cost(0);
 5343   format %{ %}
 5344   interface(REG_INTER);
 5345 %}
 5346 
 5347 operand vRegD_V12()
 5348 %{
 5349   constraint(ALLOC_IN_RC(v12_reg));
 5350   match(RegD);
 5351   op_cost(0);
 5352   format %{ %}
 5353   interface(REG_INTER);
 5354 %}
 5355 
 5356 operand vRegD_V13()
 5357 %{
 5358   constraint(ALLOC_IN_RC(v13_reg));
 5359   match(RegD);
 5360   op_cost(0);
 5361   format %{ %}
 5362   interface(REG_INTER);
 5363 %}
 5364 
 5365 operand vRegD_V14()
 5366 %{
 5367   constraint(ALLOC_IN_RC(v14_reg));
 5368   match(RegD);
 5369   op_cost(0);
 5370   format %{ %}
 5371   interface(REG_INTER);
 5372 %}
 5373 
 5374 operand vRegD_V15()
 5375 %{
 5376   constraint(ALLOC_IN_RC(v15_reg));
 5377   match(RegD);
 5378   op_cost(0);
 5379   format %{ %}
 5380   interface(REG_INTER);
 5381 %}
 5382 
 5383 operand vRegD_V16()
 5384 %{
 5385   constraint(ALLOC_IN_RC(v16_reg));
 5386   match(RegD);
 5387   op_cost(0);
 5388   format %{ %}
 5389   interface(REG_INTER);
 5390 %}
 5391 
 5392 operand vRegD_V17()
 5393 %{
 5394   constraint(ALLOC_IN_RC(v17_reg));
 5395   match(RegD);
 5396   op_cost(0);
 5397   format %{ %}
 5398   interface(REG_INTER);
 5399 %}
 5400 
 5401 operand vRegD_V18()
 5402 %{
 5403   constraint(ALLOC_IN_RC(v18_reg));
 5404   match(RegD);
 5405   op_cost(0);
 5406   format %{ %}
 5407   interface(REG_INTER);
 5408 %}
 5409 
 5410 operand vRegD_V19()
 5411 %{
 5412   constraint(ALLOC_IN_RC(v19_reg));
 5413   match(RegD);
 5414   op_cost(0);
 5415   format %{ %}
 5416   interface(REG_INTER);
 5417 %}
 5418 
 5419 operand vRegD_V20()
 5420 %{
 5421   constraint(ALLOC_IN_RC(v20_reg));
 5422   match(RegD);
 5423   op_cost(0);
 5424   format %{ %}
 5425   interface(REG_INTER);
 5426 %}
 5427 
 5428 operand vRegD_V21()
 5429 %{
 5430   constraint(ALLOC_IN_RC(v21_reg));
 5431   match(RegD);
 5432   op_cost(0);
 5433   format %{ %}
 5434   interface(REG_INTER);
 5435 %}
 5436 
 5437 operand vRegD_V22()
 5438 %{
 5439   constraint(ALLOC_IN_RC(v22_reg));
 5440   match(RegD);
 5441   op_cost(0);
 5442   format %{ %}
 5443   interface(REG_INTER);
 5444 %}
 5445 
 5446 operand vRegD_V23()
 5447 %{
 5448   constraint(ALLOC_IN_RC(v23_reg));
 5449   match(RegD);
 5450   op_cost(0);
 5451   format %{ %}
 5452   interface(REG_INTER);
 5453 %}
 5454 
 5455 operand vRegD_V24()
 5456 %{
 5457   constraint(ALLOC_IN_RC(v24_reg));
 5458   match(RegD);
 5459   op_cost(0);
 5460   format %{ %}
 5461   interface(REG_INTER);
 5462 %}
 5463 
 5464 operand vRegD_V25()
 5465 %{
 5466   constraint(ALLOC_IN_RC(v25_reg));
 5467   match(RegD);
 5468   op_cost(0);
 5469   format %{ %}
 5470   interface(REG_INTER);
 5471 %}
 5472 
 5473 operand vRegD_V26()
 5474 %{
 5475   constraint(ALLOC_IN_RC(v26_reg));
 5476   match(RegD);
 5477   op_cost(0);
 5478   format %{ %}
 5479   interface(REG_INTER);
 5480 %}
 5481 
 5482 operand vRegD_V27()
 5483 %{
 5484   constraint(ALLOC_IN_RC(v27_reg));
 5485   match(RegD);
 5486   op_cost(0);
 5487   format %{ %}
 5488   interface(REG_INTER);
 5489 %}
 5490 
 5491 operand vRegD_V28()
 5492 %{
 5493   constraint(ALLOC_IN_RC(v28_reg));
 5494   match(RegD);
 5495   op_cost(0);
 5496   format %{ %}
 5497   interface(REG_INTER);
 5498 %}
 5499 
 5500 operand vRegD_V29()
 5501 %{
 5502   constraint(ALLOC_IN_RC(v29_reg));
 5503   match(RegD);
 5504   op_cost(0);
 5505   format %{ %}
 5506   interface(REG_INTER);
 5507 %}
 5508 
 5509 operand vRegD_V30()
 5510 %{
 5511   constraint(ALLOC_IN_RC(v30_reg));
 5512   match(RegD);
 5513   op_cost(0);
 5514   format %{ %}
 5515   interface(REG_INTER);
 5516 %}
 5517 
 5518 operand vRegD_V31()
 5519 %{
 5520   constraint(ALLOC_IN_RC(v31_reg));
 5521   match(RegD);
 5522   op_cost(0);
 5523   format %{ %}
 5524   interface(REG_INTER);
 5525 %}
 5526 
 5527 operand pReg()
 5528 %{
 5529   constraint(ALLOC_IN_RC(pr_reg));
 5530   match(RegVectMask);
 5531   op_cost(0);
 5532   format %{ %}
 5533   interface(REG_INTER);
 5534 %}
 5535 
 5536 operand pRegGov()
 5537 %{
 5538   constraint(ALLOC_IN_RC(gov_pr));
 5539   match(RegVectMask);
 5540   op_cost(0);
 5541   format %{ %}
 5542   interface(REG_INTER);
 5543 %}
 5544 
 5545 // Flags register, used as output of signed compare instructions
 5546 
 5547 // note that on AArch64 we also use this register as the output for
 5548 // for floating point compare instructions (CmpF CmpD). this ensures
 5549 // that ordered inequality tests use GT, GE, LT or LE none of which
 5550 // pass through cases where the result is unordered i.e. one or both
 5551 // inputs to the compare is a NaN. this means that the ideal code can
 5552 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5553 // (where the comparison should always fail). EQ and NE tests are
 5554 // always generated in ideal code so that unordered folds into the NE
 5555 // case, matching the behaviour of AArch64 NE.
 5556 //
 5557 // This differs from x86 where the outputs of FP compares use a
 5558 // special FP flags registers and where compares based on this
 5559 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5560 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5561 // to explicitly handle the unordered case in branches. x86 also has
 5562 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5563 
 5564 operand rFlagsReg()
 5565 %{
 5566   constraint(ALLOC_IN_RC(int_flags));
 5567   match(RegFlags);
 5568 
 5569   op_cost(0);
 5570   format %{ "RFLAGS" %}
 5571   interface(REG_INTER);
 5572 %}
 5573 
 5574 // Flags register, used as output of unsigned compare instructions
 5575 operand rFlagsRegU()
 5576 %{
 5577   constraint(ALLOC_IN_RC(int_flags));
 5578   match(RegFlags);
 5579 
 5580   op_cost(0);
 5581   format %{ "RFLAGSU" %}
 5582   interface(REG_INTER);
 5583 %}
 5584 
 5585 // Special Registers
 5586 
 5587 // Method Register
 5588 operand inline_cache_RegP(iRegP reg)
 5589 %{
 5590   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5591   match(reg);
 5592   match(iRegPNoSp);
 5593   op_cost(0);
 5594   format %{ %}
 5595   interface(REG_INTER);
 5596 %}
 5597 
 5598 // Thread Register
 5599 operand thread_RegP(iRegP reg)
 5600 %{
 5601   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5602   match(reg);
 5603   op_cost(0);
 5604   format %{ %}
 5605   interface(REG_INTER);
 5606 %}
 5607 
 5608 operand lr_RegP(iRegP reg)
 5609 %{
 5610   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5611   match(reg);
 5612   op_cost(0);
 5613   format %{ %}
 5614   interface(REG_INTER);
 5615 %}
 5616 
 5617 //----------Memory Operands----------------------------------------------------
 5618 
 5619 operand indirect(iRegP reg)
 5620 %{
 5621   constraint(ALLOC_IN_RC(ptr_reg));
 5622   match(reg);
 5623   op_cost(0);
 5624   format %{ "[$reg]" %}
 5625   interface(MEMORY_INTER) %{
 5626     base($reg);
 5627     index(0xffffffff);
 5628     scale(0x0);
 5629     disp(0x0);
 5630   %}
 5631 %}
 5632 
 5633 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5634 %{
 5635   constraint(ALLOC_IN_RC(ptr_reg));
 5636   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5637   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5638   op_cost(0);
 5639   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5640   interface(MEMORY_INTER) %{
 5641     base($reg);
 5642     index($ireg);
 5643     scale($scale);
 5644     disp(0x0);
 5645   %}
 5646 %}
 5647 
 5648 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5649 %{
 5650   constraint(ALLOC_IN_RC(ptr_reg));
 5651   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5652   match(AddP reg (LShiftL lreg scale));
 5653   op_cost(0);
 5654   format %{ "$reg, $lreg lsl($scale)" %}
 5655   interface(MEMORY_INTER) %{
 5656     base($reg);
 5657     index($lreg);
 5658     scale($scale);
 5659     disp(0x0);
 5660   %}
 5661 %}
 5662 
 5663 operand indIndexI2L(iRegP reg, iRegI ireg)
 5664 %{
 5665   constraint(ALLOC_IN_RC(ptr_reg));
 5666   match(AddP reg (ConvI2L ireg));
 5667   op_cost(0);
 5668   format %{ "$reg, $ireg, 0, I2L" %}
 5669   interface(MEMORY_INTER) %{
 5670     base($reg);
 5671     index($ireg);
 5672     scale(0x0);
 5673     disp(0x0);
 5674   %}
 5675 %}
 5676 
 5677 operand indIndex(iRegP reg, iRegL lreg)
 5678 %{
 5679   constraint(ALLOC_IN_RC(ptr_reg));
 5680   match(AddP reg lreg);
 5681   op_cost(0);
 5682   format %{ "$reg, $lreg" %}
 5683   interface(MEMORY_INTER) %{
 5684     base($reg);
 5685     index($lreg);
 5686     scale(0x0);
 5687     disp(0x0);
 5688   %}
 5689 %}
 5690 
 5691 operand indOffI(iRegP reg, immIOffset off)
 5692 %{
 5693   constraint(ALLOC_IN_RC(ptr_reg));
 5694   match(AddP reg off);
 5695   op_cost(0);
 5696   format %{ "[$reg, $off]" %}
 5697   interface(MEMORY_INTER) %{
 5698     base($reg);
 5699     index(0xffffffff);
 5700     scale(0x0);
 5701     disp($off);
 5702   %}
 5703 %}
 5704 
 5705 operand indOffI1(iRegP reg, immIOffset1 off)
 5706 %{
 5707   constraint(ALLOC_IN_RC(ptr_reg));
 5708   match(AddP reg off);
 5709   op_cost(0);
 5710   format %{ "[$reg, $off]" %}
 5711   interface(MEMORY_INTER) %{
 5712     base($reg);
 5713     index(0xffffffff);
 5714     scale(0x0);
 5715     disp($off);
 5716   %}
 5717 %}
 5718 
 5719 operand indOffI2(iRegP reg, immIOffset2 off)
 5720 %{
 5721   constraint(ALLOC_IN_RC(ptr_reg));
 5722   match(AddP reg off);
 5723   op_cost(0);
 5724   format %{ "[$reg, $off]" %}
 5725   interface(MEMORY_INTER) %{
 5726     base($reg);
 5727     index(0xffffffff);
 5728     scale(0x0);
 5729     disp($off);
 5730   %}
 5731 %}
 5732 
 5733 operand indOffI4(iRegP reg, immIOffset4 off)
 5734 %{
 5735   constraint(ALLOC_IN_RC(ptr_reg));
 5736   match(AddP reg off);
 5737   op_cost(0);
 5738   format %{ "[$reg, $off]" %}
 5739   interface(MEMORY_INTER) %{
 5740     base($reg);
 5741     index(0xffffffff);
 5742     scale(0x0);
 5743     disp($off);
 5744   %}
 5745 %}
 5746 
 5747 operand indOffI8(iRegP reg, immIOffset8 off)
 5748 %{
 5749   constraint(ALLOC_IN_RC(ptr_reg));
 5750   match(AddP reg off);
 5751   op_cost(0);
 5752   format %{ "[$reg, $off]" %}
 5753   interface(MEMORY_INTER) %{
 5754     base($reg);
 5755     index(0xffffffff);
 5756     scale(0x0);
 5757     disp($off);
 5758   %}
 5759 %}
 5760 
 5761 operand indOffI16(iRegP reg, immIOffset16 off)
 5762 %{
 5763   constraint(ALLOC_IN_RC(ptr_reg));
 5764   match(AddP reg off);
 5765   op_cost(0);
 5766   format %{ "[$reg, $off]" %}
 5767   interface(MEMORY_INTER) %{
 5768     base($reg);
 5769     index(0xffffffff);
 5770     scale(0x0);
 5771     disp($off);
 5772   %}
 5773 %}
 5774 
 5775 operand indOffL(iRegP reg, immLoffset off)
 5776 %{
 5777   constraint(ALLOC_IN_RC(ptr_reg));
 5778   match(AddP reg off);
 5779   op_cost(0);
 5780   format %{ "[$reg, $off]" %}
 5781   interface(MEMORY_INTER) %{
 5782     base($reg);
 5783     index(0xffffffff);
 5784     scale(0x0);
 5785     disp($off);
 5786   %}
 5787 %}
 5788 
 5789 operand indOffL1(iRegP reg, immLoffset1 off)
 5790 %{
 5791   constraint(ALLOC_IN_RC(ptr_reg));
 5792   match(AddP reg off);
 5793   op_cost(0);
 5794   format %{ "[$reg, $off]" %}
 5795   interface(MEMORY_INTER) %{
 5796     base($reg);
 5797     index(0xffffffff);
 5798     scale(0x0);
 5799     disp($off);
 5800   %}
 5801 %}
 5802 
 5803 operand indOffL2(iRegP reg, immLoffset2 off)
 5804 %{
 5805   constraint(ALLOC_IN_RC(ptr_reg));
 5806   match(AddP reg off);
 5807   op_cost(0);
 5808   format %{ "[$reg, $off]" %}
 5809   interface(MEMORY_INTER) %{
 5810     base($reg);
 5811     index(0xffffffff);
 5812     scale(0x0);
 5813     disp($off);
 5814   %}
 5815 %}
 5816 
 5817 operand indOffL4(iRegP reg, immLoffset4 off)
 5818 %{
 5819   constraint(ALLOC_IN_RC(ptr_reg));
 5820   match(AddP reg off);
 5821   op_cost(0);
 5822   format %{ "[$reg, $off]" %}
 5823   interface(MEMORY_INTER) %{
 5824     base($reg);
 5825     index(0xffffffff);
 5826     scale(0x0);
 5827     disp($off);
 5828   %}
 5829 %}
 5830 
 5831 operand indOffL8(iRegP reg, immLoffset8 off)
 5832 %{
 5833   constraint(ALLOC_IN_RC(ptr_reg));
 5834   match(AddP reg off);
 5835   op_cost(0);
 5836   format %{ "[$reg, $off]" %}
 5837   interface(MEMORY_INTER) %{
 5838     base($reg);
 5839     index(0xffffffff);
 5840     scale(0x0);
 5841     disp($off);
 5842   %}
 5843 %}
 5844 
 5845 operand indOffL16(iRegP reg, immLoffset16 off)
 5846 %{
 5847   constraint(ALLOC_IN_RC(ptr_reg));
 5848   match(AddP reg off);
 5849   op_cost(0);
 5850   format %{ "[$reg, $off]" %}
 5851   interface(MEMORY_INTER) %{
 5852     base($reg);
 5853     index(0xffffffff);
 5854     scale(0x0);
 5855     disp($off);
 5856   %}
 5857 %}
 5858 
 5859 operand indirectN(iRegN reg)
 5860 %{
 5861   predicate(CompressedOops::shift() == 0);
 5862   constraint(ALLOC_IN_RC(ptr_reg));
 5863   match(DecodeN reg);
 5864   op_cost(0);
 5865   format %{ "[$reg]\t# narrow" %}
 5866   interface(MEMORY_INTER) %{
 5867     base($reg);
 5868     index(0xffffffff);
 5869     scale(0x0);
 5870     disp(0x0);
 5871   %}
 5872 %}
 5873 
 5874 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5875 %{
 5876   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5877   constraint(ALLOC_IN_RC(ptr_reg));
 5878   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5879   op_cost(0);
 5880   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5881   interface(MEMORY_INTER) %{
 5882     base($reg);
 5883     index($ireg);
 5884     scale($scale);
 5885     disp(0x0);
 5886   %}
 5887 %}
 5888 
 5889 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5890 %{
 5891   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5892   constraint(ALLOC_IN_RC(ptr_reg));
 5893   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5894   op_cost(0);
 5895   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5896   interface(MEMORY_INTER) %{
 5897     base($reg);
 5898     index($lreg);
 5899     scale($scale);
 5900     disp(0x0);
 5901   %}
 5902 %}
 5903 
 5904 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5905 %{
 5906   predicate(CompressedOops::shift() == 0);
 5907   constraint(ALLOC_IN_RC(ptr_reg));
 5908   match(AddP (DecodeN reg) (ConvI2L ireg));
 5909   op_cost(0);
 5910   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5911   interface(MEMORY_INTER) %{
 5912     base($reg);
 5913     index($ireg);
 5914     scale(0x0);
 5915     disp(0x0);
 5916   %}
 5917 %}
 5918 
 5919 operand indIndexN(iRegN reg, iRegL lreg)
 5920 %{
 5921   predicate(CompressedOops::shift() == 0);
 5922   constraint(ALLOC_IN_RC(ptr_reg));
 5923   match(AddP (DecodeN reg) lreg);
 5924   op_cost(0);
 5925   format %{ "$reg, $lreg\t# narrow" %}
 5926   interface(MEMORY_INTER) %{
 5927     base($reg);
 5928     index($lreg);
 5929     scale(0x0);
 5930     disp(0x0);
 5931   %}
 5932 %}
 5933 
 5934 operand indOffIN(iRegN reg, immIOffset off)
 5935 %{
 5936   predicate(CompressedOops::shift() == 0);
 5937   constraint(ALLOC_IN_RC(ptr_reg));
 5938   match(AddP (DecodeN reg) off);
 5939   op_cost(0);
 5940   format %{ "[$reg, $off]\t# narrow" %}
 5941   interface(MEMORY_INTER) %{
 5942     base($reg);
 5943     index(0xffffffff);
 5944     scale(0x0);
 5945     disp($off);
 5946   %}
 5947 %}
 5948 
 5949 operand indOffLN(iRegN reg, immLoffset off)
 5950 %{
 5951   predicate(CompressedOops::shift() == 0);
 5952   constraint(ALLOC_IN_RC(ptr_reg));
 5953   match(AddP (DecodeN reg) off);
 5954   op_cost(0);
 5955   format %{ "[$reg, $off]\t# narrow" %}
 5956   interface(MEMORY_INTER) %{
 5957     base($reg);
 5958     index(0xffffffff);
 5959     scale(0x0);
 5960     disp($off);
 5961   %}
 5962 %}
 5963 
 5964 
 5965 
 5966 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 5967 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 5968 %{
 5969   constraint(ALLOC_IN_RC(ptr_reg));
 5970   match(AddP reg off);
 5971   op_cost(0);
 5972   format %{ "[$reg, $off]" %}
 5973   interface(MEMORY_INTER) %{
 5974     base($reg);
 5975     index(0xffffffff);
 5976     scale(0x0);
 5977     disp($off);
 5978   %}
 5979 %}
 5980 
 5981 //----------Special Memory Operands--------------------------------------------
 5982 // Stack Slot Operand - This operand is used for loading and storing temporary
 5983 //                      values on the stack where a match requires a value to
 5984 //                      flow through memory.
 5985 operand stackSlotP(sRegP reg)
 5986 %{
 5987   constraint(ALLOC_IN_RC(stack_slots));
 5988   op_cost(100);
 5989   // No match rule because this operand is only generated in matching
 5990   // match(RegP);
 5991   format %{ "[$reg]" %}
 5992   interface(MEMORY_INTER) %{
 5993     base(0x1e);  // RSP
 5994     index(0x0);  // No Index
 5995     scale(0x0);  // No Scale
 5996     disp($reg);  // Stack Offset
 5997   %}
 5998 %}
 5999 
 6000 operand stackSlotI(sRegI reg)
 6001 %{
 6002   constraint(ALLOC_IN_RC(stack_slots));
 6003   // No match rule because this operand is only generated in matching
 6004   // match(RegI);
 6005   format %{ "[$reg]" %}
 6006   interface(MEMORY_INTER) %{
 6007     base(0x1e);  // RSP
 6008     index(0x0);  // No Index
 6009     scale(0x0);  // No Scale
 6010     disp($reg);  // Stack Offset
 6011   %}
 6012 %}
 6013 
 6014 operand stackSlotF(sRegF reg)
 6015 %{
 6016   constraint(ALLOC_IN_RC(stack_slots));
 6017   // No match rule because this operand is only generated in matching
 6018   // match(RegF);
 6019   format %{ "[$reg]" %}
 6020   interface(MEMORY_INTER) %{
 6021     base(0x1e);  // RSP
 6022     index(0x0);  // No Index
 6023     scale(0x0);  // No Scale
 6024     disp($reg);  // Stack Offset
 6025   %}
 6026 %}
 6027 
 6028 operand stackSlotD(sRegD reg)
 6029 %{
 6030   constraint(ALLOC_IN_RC(stack_slots));
 6031   // No match rule because this operand is only generated in matching
 6032   // match(RegD);
 6033   format %{ "[$reg]" %}
 6034   interface(MEMORY_INTER) %{
 6035     base(0x1e);  // RSP
 6036     index(0x0);  // No Index
 6037     scale(0x0);  // No Scale
 6038     disp($reg);  // Stack Offset
 6039   %}
 6040 %}
 6041 
 6042 operand stackSlotL(sRegL reg)
 6043 %{
 6044   constraint(ALLOC_IN_RC(stack_slots));
 6045   // No match rule because this operand is only generated in matching
 6046   // match(RegL);
 6047   format %{ "[$reg]" %}
 6048   interface(MEMORY_INTER) %{
 6049     base(0x1e);  // RSP
 6050     index(0x0);  // No Index
 6051     scale(0x0);  // No Scale
 6052     disp($reg);  // Stack Offset
 6053   %}
 6054 %}
 6055 
 6056 // Operands for expressing Control Flow
 6057 // NOTE: Label is a predefined operand which should not be redefined in
 6058 //       the AD file. It is generically handled within the ADLC.
 6059 
 6060 //----------Conditional Branch Operands----------------------------------------
 6061 // Comparison Op  - This is the operation of the comparison, and is limited to
 6062 //                  the following set of codes:
 6063 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6064 //
 6065 // Other attributes of the comparison, such as unsignedness, are specified
 6066 // by the comparison instruction that sets a condition code flags register.
 6067 // That result is represented by a flags operand whose subtype is appropriate
 6068 // to the unsignedness (etc.) of the comparison.
 6069 //
 6070 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6071 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6072 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6073 
 6074 // used for signed integral comparisons and fp comparisons
 6075 
 6076 operand cmpOp()
 6077 %{
 6078   match(Bool);
 6079 
 6080   format %{ "" %}
 6081   interface(COND_INTER) %{
 6082     equal(0x0, "eq");
 6083     not_equal(0x1, "ne");
 6084     less(0xb, "lt");
 6085     greater_equal(0xa, "ge");
 6086     less_equal(0xd, "le");
 6087     greater(0xc, "gt");
 6088     overflow(0x6, "vs");
 6089     no_overflow(0x7, "vc");
 6090   %}
 6091 %}
 6092 
 6093 // used for unsigned integral comparisons
 6094 
 6095 operand cmpOpU()
 6096 %{
 6097   match(Bool);
 6098 
 6099   format %{ "" %}
 6100   interface(COND_INTER) %{
 6101     equal(0x0, "eq");
 6102     not_equal(0x1, "ne");
 6103     less(0x3, "lo");
 6104     greater_equal(0x2, "hs");
 6105     less_equal(0x9, "ls");
 6106     greater(0x8, "hi");
 6107     overflow(0x6, "vs");
 6108     no_overflow(0x7, "vc");
 6109   %}
 6110 %}
 6111 
 6112 // used for certain integral comparisons which can be
 6113 // converted to cbxx or tbxx instructions
 6114 
 6115 operand cmpOpEqNe()
 6116 %{
 6117   match(Bool);
 6118   op_cost(0);
 6119   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6120             || n->as_Bool()->_test._test == BoolTest::eq);
 6121 
 6122   format %{ "" %}
 6123   interface(COND_INTER) %{
 6124     equal(0x0, "eq");
 6125     not_equal(0x1, "ne");
 6126     less(0xb, "lt");
 6127     greater_equal(0xa, "ge");
 6128     less_equal(0xd, "le");
 6129     greater(0xc, "gt");
 6130     overflow(0x6, "vs");
 6131     no_overflow(0x7, "vc");
 6132   %}
 6133 %}
 6134 
 6135 // used for certain integral comparisons which can be
 6136 // converted to cbxx or tbxx instructions
 6137 
 6138 operand cmpOpLtGe()
 6139 %{
 6140   match(Bool);
 6141   op_cost(0);
 6142 
 6143   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6144             || n->as_Bool()->_test._test == BoolTest::ge);
 6145 
 6146   format %{ "" %}
 6147   interface(COND_INTER) %{
 6148     equal(0x0, "eq");
 6149     not_equal(0x1, "ne");
 6150     less(0xb, "lt");
 6151     greater_equal(0xa, "ge");
 6152     less_equal(0xd, "le");
 6153     greater(0xc, "gt");
 6154     overflow(0x6, "vs");
 6155     no_overflow(0x7, "vc");
 6156   %}
 6157 %}
 6158 
 6159 // used for certain unsigned integral comparisons which can be
 6160 // converted to cbxx or tbxx instructions
 6161 
 6162 operand cmpOpUEqNeLtGe()
 6163 %{
 6164   match(Bool);
 6165   op_cost(0);
 6166 
 6167   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6168             || n->as_Bool()->_test._test == BoolTest::ne
 6169             || n->as_Bool()->_test._test == BoolTest::lt
 6170             || n->as_Bool()->_test._test == BoolTest::ge);
 6171 
 6172   format %{ "" %}
 6173   interface(COND_INTER) %{
 6174     equal(0x0, "eq");
 6175     not_equal(0x1, "ne");
 6176     less(0xb, "lt");
 6177     greater_equal(0xa, "ge");
 6178     less_equal(0xd, "le");
 6179     greater(0xc, "gt");
 6180     overflow(0x6, "vs");
 6181     no_overflow(0x7, "vc");
 6182   %}
 6183 %}
 6184 
 6185 // Special operand allowing long args to int ops to be truncated for free
 6186 
 6187 operand iRegL2I(iRegL reg) %{
 6188 
 6189   op_cost(0);
 6190 
 6191   match(ConvL2I reg);
 6192 
 6193   format %{ "l2i($reg)" %}
 6194 
 6195   interface(REG_INTER)
 6196 %}
 6197 
 6198 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6199 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6200 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6201 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6202 
 6203 //----------OPERAND CLASSES----------------------------------------------------
 6204 // Operand Classes are groups of operands that are used as to simplify
 6205 // instruction definitions by not requiring the AD writer to specify
 6206 // separate instructions for every form of operand when the
 6207 // instruction accepts multiple operand types with the same basic
 6208 // encoding and format. The classic case of this is memory operands.
 6209 
 6210 // memory is used to define read/write location for load/store
 6211 // instruction defs. we can turn a memory op into an Address
 6212 
 6213 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6214                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6215 
 6216 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6217                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6218 
 6219 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6220                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6221 
 6222 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6223                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6224 
 6225 // All of the memory operands. For the pipeline description.
 6226 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6227                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6228                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6229 
 6230 
 6231 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6232 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6233 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6234 // can be elided because the 32-bit instruction will just employ the
 6235 // lower 32 bits anyway.
 6236 //
 6237 // n.b. this does not elide all L2I conversions. if the truncated
 6238 // value is consumed by more than one operation then the ConvL2I
 6239 // cannot be bundled into the consuming nodes so an l2i gets planted
 6240 // (actually a movw $dst $src) and the downstream instructions consume
 6241 // the result of the l2i as an iRegI input. That's a shame since the
 6242 // movw is actually redundant but its not too costly.
 6243 
 6244 opclass iRegIorL2I(iRegI, iRegL2I);
 6245 
 6246 //----------PIPELINE-----------------------------------------------------------
 6247 // Rules which define the behavior of the target architectures pipeline.
 6248 
 6249 // For specific pipelines, eg A53, define the stages of that pipeline
 6250 //pipe_desc(ISS, EX1, EX2, WR);
 6251 #define ISS S0
 6252 #define EX1 S1
 6253 #define EX2 S2
 6254 #define WR  S3
 6255 
 6256 // Integer ALU reg operation
 6257 pipeline %{
 6258 
 6259 attributes %{
 6260   // ARM instructions are of fixed length
 6261   fixed_size_instructions;        // Fixed size instructions TODO does
 6262   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6263   // ARM instructions come in 32-bit word units
 6264   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6265   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6266   instruction_fetch_units = 1;       // of 64 bytes
 6267 
 6268   // List of nop instructions
 6269   nops( MachNop );
 6270 %}
 6271 
 6272 // We don't use an actual pipeline model so don't care about resources
 6273 // or description. we do use pipeline classes to introduce fixed
 6274 // latencies
 6275 
 6276 //----------RESOURCES----------------------------------------------------------
 6277 // Resources are the functional units available to the machine
 6278 
 6279 resources( INS0, INS1, INS01 = INS0 | INS1,
 6280            ALU0, ALU1, ALU = ALU0 | ALU1,
 6281            MAC,
 6282            DIV,
 6283            BRANCH,
 6284            LDST,
 6285            NEON_FP);
 6286 
 6287 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6288 // Pipeline Description specifies the stages in the machine's pipeline
 6289 
 6290 // Define the pipeline as a generic 6 stage pipeline
 6291 pipe_desc(S0, S1, S2, S3, S4, S5);
 6292 
 6293 //----------PIPELINE CLASSES---------------------------------------------------
 6294 // Pipeline Classes describe the stages in which input and output are
 6295 // referenced by the hardware pipeline.
 6296 
 6297 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6298 %{
 6299   single_instruction;
 6300   src1   : S1(read);
 6301   src2   : S2(read);
 6302   dst    : S5(write);
 6303   INS01  : ISS;
 6304   NEON_FP : S5;
 6305 %}
 6306 
 6307 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6308 %{
 6309   single_instruction;
 6310   src1   : S1(read);
 6311   src2   : S2(read);
 6312   dst    : S5(write);
 6313   INS01  : ISS;
 6314   NEON_FP : S5;
 6315 %}
 6316 
 6317 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6318 %{
 6319   single_instruction;
 6320   src    : S1(read);
 6321   dst    : S5(write);
 6322   INS01  : ISS;
 6323   NEON_FP : S5;
 6324 %}
 6325 
 6326 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6327 %{
 6328   single_instruction;
 6329   src    : S1(read);
 6330   dst    : S5(write);
 6331   INS01  : ISS;
 6332   NEON_FP : S5;
 6333 %}
 6334 
 6335 pipe_class fp_d2f(vRegF dst, vRegD src)
 6336 %{
 6337   single_instruction;
 6338   src    : S1(read);
 6339   dst    : S5(write);
 6340   INS01  : ISS;
 6341   NEON_FP : S5;
 6342 %}
 6343 
 6344 pipe_class fp_f2d(vRegD dst, vRegF src)
 6345 %{
 6346   single_instruction;
 6347   src    : S1(read);
 6348   dst    : S5(write);
 6349   INS01  : ISS;
 6350   NEON_FP : S5;
 6351 %}
 6352 
 6353 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6354 %{
 6355   single_instruction;
 6356   src    : S1(read);
 6357   dst    : S5(write);
 6358   INS01  : ISS;
 6359   NEON_FP : S5;
 6360 %}
 6361 
 6362 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6363 %{
 6364   single_instruction;
 6365   src    : S1(read);
 6366   dst    : S5(write);
 6367   INS01  : ISS;
 6368   NEON_FP : S5;
 6369 %}
 6370 
 6371 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6372 %{
 6373   single_instruction;
 6374   src    : S1(read);
 6375   dst    : S5(write);
 6376   INS01  : ISS;
 6377   NEON_FP : S5;
 6378 %}
 6379 
 6380 pipe_class fp_l2f(vRegF dst, iRegL src)
 6381 %{
 6382   single_instruction;
 6383   src    : S1(read);
 6384   dst    : S5(write);
 6385   INS01  : ISS;
 6386   NEON_FP : S5;
 6387 %}
 6388 
 6389 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6390 %{
 6391   single_instruction;
 6392   src    : S1(read);
 6393   dst    : S5(write);
 6394   INS01  : ISS;
 6395   NEON_FP : S5;
 6396 %}
 6397 
 6398 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6399 %{
 6400   single_instruction;
 6401   src    : S1(read);
 6402   dst    : S5(write);
 6403   INS01  : ISS;
 6404   NEON_FP : S5;
 6405 %}
 6406 
 6407 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6408 %{
 6409   single_instruction;
 6410   src    : S1(read);
 6411   dst    : S5(write);
 6412   INS01  : ISS;
 6413   NEON_FP : S5;
 6414 %}
 6415 
 6416 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6417 %{
 6418   single_instruction;
 6419   src    : S1(read);
 6420   dst    : S5(write);
 6421   INS01  : ISS;
 6422   NEON_FP : S5;
 6423 %}
 6424 
 6425 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6426 %{
 6427   single_instruction;
 6428   src1   : S1(read);
 6429   src2   : S2(read);
 6430   dst    : S5(write);
 6431   INS0   : ISS;
 6432   NEON_FP : S5;
 6433 %}
 6434 
 6435 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6436 %{
 6437   single_instruction;
 6438   src1   : S1(read);
 6439   src2   : S2(read);
 6440   dst    : S5(write);
 6441   INS0   : ISS;
 6442   NEON_FP : S5;
 6443 %}
 6444 
 6445 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6446 %{
 6447   single_instruction;
 6448   cr     : S1(read);
 6449   src1   : S1(read);
 6450   src2   : S1(read);
 6451   dst    : S3(write);
 6452   INS01  : ISS;
 6453   NEON_FP : S3;
 6454 %}
 6455 
 6456 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6457 %{
 6458   single_instruction;
 6459   cr     : S1(read);
 6460   src1   : S1(read);
 6461   src2   : S1(read);
 6462   dst    : S3(write);
 6463   INS01  : ISS;
 6464   NEON_FP : S3;
 6465 %}
 6466 
 6467 pipe_class fp_imm_s(vRegF dst)
 6468 %{
 6469   single_instruction;
 6470   dst    : S3(write);
 6471   INS01  : ISS;
 6472   NEON_FP : S3;
 6473 %}
 6474 
 6475 pipe_class fp_imm_d(vRegD dst)
 6476 %{
 6477   single_instruction;
 6478   dst    : S3(write);
 6479   INS01  : ISS;
 6480   NEON_FP : S3;
 6481 %}
 6482 
 6483 pipe_class fp_load_constant_s(vRegF dst)
 6484 %{
 6485   single_instruction;
 6486   dst    : S4(write);
 6487   INS01  : ISS;
 6488   NEON_FP : S4;
 6489 %}
 6490 
 6491 pipe_class fp_load_constant_d(vRegD dst)
 6492 %{
 6493   single_instruction;
 6494   dst    : S4(write);
 6495   INS01  : ISS;
 6496   NEON_FP : S4;
 6497 %}
 6498 
 6499 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6500 %{
 6501   single_instruction;
 6502   dst    : S5(write);
 6503   src1   : S1(read);
 6504   src2   : S1(read);
 6505   INS01  : ISS;
 6506   NEON_FP : S5;
 6507 %}
 6508 
 6509 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6510 %{
 6511   single_instruction;
 6512   dst    : S5(write);
 6513   src1   : S1(read);
 6514   src2   : S1(read);
 6515   INS0   : ISS;
 6516   NEON_FP : S5;
 6517 %}
 6518 
 6519 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6520 %{
 6521   single_instruction;
 6522   dst    : S5(write);
 6523   src1   : S1(read);
 6524   src2   : S1(read);
 6525   dst    : S1(read);
 6526   INS01  : ISS;
 6527   NEON_FP : S5;
 6528 %}
 6529 
 6530 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6531 %{
 6532   single_instruction;
 6533   dst    : S5(write);
 6534   src1   : S1(read);
 6535   src2   : S1(read);
 6536   dst    : S1(read);
 6537   INS0   : ISS;
 6538   NEON_FP : S5;
 6539 %}
 6540 
 6541 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6542 %{
 6543   single_instruction;
 6544   dst    : S4(write);
 6545   src1   : S2(read);
 6546   src2   : S2(read);
 6547   INS01  : ISS;
 6548   NEON_FP : S4;
 6549 %}
 6550 
 6551 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6552 %{
 6553   single_instruction;
 6554   dst    : S4(write);
 6555   src1   : S2(read);
 6556   src2   : S2(read);
 6557   INS0   : ISS;
 6558   NEON_FP : S4;
 6559 %}
 6560 
 6561 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6562 %{
 6563   single_instruction;
 6564   dst    : S3(write);
 6565   src1   : S2(read);
 6566   src2   : S2(read);
 6567   INS01  : ISS;
 6568   NEON_FP : S3;
 6569 %}
 6570 
 6571 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6572 %{
 6573   single_instruction;
 6574   dst    : S3(write);
 6575   src1   : S2(read);
 6576   src2   : S2(read);
 6577   INS0   : ISS;
 6578   NEON_FP : S3;
 6579 %}
 6580 
 6581 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6582 %{
 6583   single_instruction;
 6584   dst    : S3(write);
 6585   src    : S1(read);
 6586   shift  : S1(read);
 6587   INS01  : ISS;
 6588   NEON_FP : S3;
 6589 %}
 6590 
 6591 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6592 %{
 6593   single_instruction;
 6594   dst    : S3(write);
 6595   src    : S1(read);
 6596   shift  : S1(read);
 6597   INS0   : ISS;
 6598   NEON_FP : S3;
 6599 %}
 6600 
 6601 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6602 %{
 6603   single_instruction;
 6604   dst    : S3(write);
 6605   src    : S1(read);
 6606   INS01  : ISS;
 6607   NEON_FP : S3;
 6608 %}
 6609 
 6610 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6611 %{
 6612   single_instruction;
 6613   dst    : S3(write);
 6614   src    : S1(read);
 6615   INS0   : ISS;
 6616   NEON_FP : S3;
 6617 %}
 6618 
 6619 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6620 %{
 6621   single_instruction;
 6622   dst    : S5(write);
 6623   src1   : S1(read);
 6624   src2   : S1(read);
 6625   INS01  : ISS;
 6626   NEON_FP : S5;
 6627 %}
 6628 
 6629 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6630 %{
 6631   single_instruction;
 6632   dst    : S5(write);
 6633   src1   : S1(read);
 6634   src2   : S1(read);
 6635   INS0   : ISS;
 6636   NEON_FP : S5;
 6637 %}
 6638 
 6639 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6640 %{
 6641   single_instruction;
 6642   dst    : S5(write);
 6643   src1   : S1(read);
 6644   src2   : S1(read);
 6645   INS0   : ISS;
 6646   NEON_FP : S5;
 6647 %}
 6648 
 6649 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6650 %{
 6651   single_instruction;
 6652   dst    : S5(write);
 6653   src1   : S1(read);
 6654   src2   : S1(read);
 6655   INS0   : ISS;
 6656   NEON_FP : S5;
 6657 %}
 6658 
 6659 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6660 %{
 6661   single_instruction;
 6662   dst    : S5(write);
 6663   src    : S1(read);
 6664   INS0   : ISS;
 6665   NEON_FP : S5;
 6666 %}
 6667 
 6668 pipe_class vunop_fp64(vecD dst, vecD src)
 6669 %{
 6670   single_instruction;
 6671   dst    : S5(write);
 6672   src    : S1(read);
 6673   INS01  : ISS;
 6674   NEON_FP : S5;
 6675 %}
 6676 
 6677 pipe_class vunop_fp128(vecX dst, vecX src)
 6678 %{
 6679   single_instruction;
 6680   dst    : S5(write);
 6681   src    : S1(read);
 6682   INS0   : ISS;
 6683   NEON_FP : S5;
 6684 %}
 6685 
 6686 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6687 %{
 6688   single_instruction;
 6689   dst    : S3(write);
 6690   src    : S1(read);
 6691   INS01  : ISS;
 6692   NEON_FP : S3;
 6693 %}
 6694 
 6695 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6696 %{
 6697   single_instruction;
 6698   dst    : S3(write);
 6699   src    : S1(read);
 6700   INS01  : ISS;
 6701   NEON_FP : S3;
 6702 %}
 6703 
 6704 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6705 %{
 6706   single_instruction;
 6707   dst    : S3(write);
 6708   src    : S1(read);
 6709   INS01  : ISS;
 6710   NEON_FP : S3;
 6711 %}
 6712 
 6713 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6714 %{
 6715   single_instruction;
 6716   dst    : S3(write);
 6717   src    : S1(read);
 6718   INS01  : ISS;
 6719   NEON_FP : S3;
 6720 %}
 6721 
 6722 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6723 %{
 6724   single_instruction;
 6725   dst    : S3(write);
 6726   src    : S1(read);
 6727   INS01  : ISS;
 6728   NEON_FP : S3;
 6729 %}
 6730 
 6731 pipe_class vmovi_reg_imm64(vecD dst)
 6732 %{
 6733   single_instruction;
 6734   dst    : S3(write);
 6735   INS01  : ISS;
 6736   NEON_FP : S3;
 6737 %}
 6738 
 6739 pipe_class vmovi_reg_imm128(vecX dst)
 6740 %{
 6741   single_instruction;
 6742   dst    : S3(write);
 6743   INS0   : ISS;
 6744   NEON_FP : S3;
 6745 %}
 6746 
 6747 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6748 %{
 6749   single_instruction;
 6750   dst    : S5(write);
 6751   mem    : ISS(read);
 6752   INS01  : ISS;
 6753   NEON_FP : S3;
 6754 %}
 6755 
 6756 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6757 %{
 6758   single_instruction;
 6759   dst    : S5(write);
 6760   mem    : ISS(read);
 6761   INS01  : ISS;
 6762   NEON_FP : S3;
 6763 %}
 6764 
 6765 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6766 %{
 6767   single_instruction;
 6768   mem    : ISS(read);
 6769   src    : S2(read);
 6770   INS01  : ISS;
 6771   NEON_FP : S3;
 6772 %}
 6773 
 6774 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6775 %{
 6776   single_instruction;
 6777   mem    : ISS(read);
 6778   src    : S2(read);
 6779   INS01  : ISS;
 6780   NEON_FP : S3;
 6781 %}
 6782 
 6783 //------- Integer ALU operations --------------------------
 6784 
 6785 // Integer ALU reg-reg operation
 6786 // Operands needed in EX1, result generated in EX2
 6787 // Eg.  ADD     x0, x1, x2
 6788 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6789 %{
 6790   single_instruction;
 6791   dst    : EX2(write);
 6792   src1   : EX1(read);
 6793   src2   : EX1(read);
 6794   INS01  : ISS; // Dual issue as instruction 0 or 1
 6795   ALU    : EX2;
 6796 %}
 6797 
 6798 // Integer ALU reg-reg operation with constant shift
 6799 // Shifted register must be available in LATE_ISS instead of EX1
 6800 // Eg.  ADD     x0, x1, x2, LSL #2
 6801 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6802 %{
 6803   single_instruction;
 6804   dst    : EX2(write);
 6805   src1   : EX1(read);
 6806   src2   : ISS(read);
 6807   INS01  : ISS;
 6808   ALU    : EX2;
 6809 %}
 6810 
 6811 // Integer ALU reg operation with constant shift
 6812 // Eg.  LSL     x0, x1, #shift
 6813 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6814 %{
 6815   single_instruction;
 6816   dst    : EX2(write);
 6817   src1   : ISS(read);
 6818   INS01  : ISS;
 6819   ALU    : EX2;
 6820 %}
 6821 
 6822 // Integer ALU reg-reg operation with variable shift
 6823 // Both operands must be available in LATE_ISS instead of EX1
 6824 // Result is available in EX1 instead of EX2
 6825 // Eg.  LSLV    x0, x1, x2
 6826 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6827 %{
 6828   single_instruction;
 6829   dst    : EX1(write);
 6830   src1   : ISS(read);
 6831   src2   : ISS(read);
 6832   INS01  : ISS;
 6833   ALU    : EX1;
 6834 %}
 6835 
 6836 // Integer ALU reg-reg operation with extract
 6837 // As for _vshift above, but result generated in EX2
 6838 // Eg.  EXTR    x0, x1, x2, #N
 6839 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6840 %{
 6841   single_instruction;
 6842   dst    : EX2(write);
 6843   src1   : ISS(read);
 6844   src2   : ISS(read);
 6845   INS1   : ISS; // Can only dual issue as Instruction 1
 6846   ALU    : EX1;
 6847 %}
 6848 
 6849 // Integer ALU reg operation
 6850 // Eg.  NEG     x0, x1
 6851 pipe_class ialu_reg(iRegI dst, iRegI src)
 6852 %{
 6853   single_instruction;
 6854   dst    : EX2(write);
 6855   src    : EX1(read);
 6856   INS01  : ISS;
 6857   ALU    : EX2;
 6858 %}
 6859 
 6860 // Integer ALU reg mmediate operation
 6861 // Eg.  ADD     x0, x1, #N
 6862 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6863 %{
 6864   single_instruction;
 6865   dst    : EX2(write);
 6866   src1   : EX1(read);
 6867   INS01  : ISS;
 6868   ALU    : EX2;
 6869 %}
 6870 
 6871 // Integer ALU immediate operation (no source operands)
 6872 // Eg.  MOV     x0, #N
 6873 pipe_class ialu_imm(iRegI dst)
 6874 %{
 6875   single_instruction;
 6876   dst    : EX1(write);
 6877   INS01  : ISS;
 6878   ALU    : EX1;
 6879 %}
 6880 
 6881 //------- Compare operation -------------------------------
 6882 
 6883 // Compare reg-reg
 6884 // Eg.  CMP     x0, x1
 6885 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6886 %{
 6887   single_instruction;
 6888 //  fixed_latency(16);
 6889   cr     : EX2(write);
 6890   op1    : EX1(read);
 6891   op2    : EX1(read);
 6892   INS01  : ISS;
 6893   ALU    : EX2;
 6894 %}
 6895 
 6896 // Compare reg-reg
 6897 // Eg.  CMP     x0, #N
 6898 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6899 %{
 6900   single_instruction;
 6901 //  fixed_latency(16);
 6902   cr     : EX2(write);
 6903   op1    : EX1(read);
 6904   INS01  : ISS;
 6905   ALU    : EX2;
 6906 %}
 6907 
 6908 //------- Conditional instructions ------------------------
 6909 
 6910 // Conditional no operands
 6911 // Eg.  CSINC   x0, zr, zr, <cond>
 6912 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6913 %{
 6914   single_instruction;
 6915   cr     : EX1(read);
 6916   dst    : EX2(write);
 6917   INS01  : ISS;
 6918   ALU    : EX2;
 6919 %}
 6920 
 6921 // Conditional 2 operand
 6922 // EG.  CSEL    X0, X1, X2, <cond>
 6923 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6924 %{
 6925   single_instruction;
 6926   cr     : EX1(read);
 6927   src1   : EX1(read);
 6928   src2   : EX1(read);
 6929   dst    : EX2(write);
 6930   INS01  : ISS;
 6931   ALU    : EX2;
 6932 %}
 6933 
 6934 // Conditional 2 operand
 6935 // EG.  CSEL    X0, X1, X2, <cond>
 6936 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6937 %{
 6938   single_instruction;
 6939   cr     : EX1(read);
 6940   src    : EX1(read);
 6941   dst    : EX2(write);
 6942   INS01  : ISS;
 6943   ALU    : EX2;
 6944 %}
 6945 
 6946 //------- Multiply pipeline operations --------------------
 6947 
 6948 // Multiply reg-reg
 6949 // Eg.  MUL     w0, w1, w2
 6950 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6951 %{
 6952   single_instruction;
 6953   dst    : WR(write);
 6954   src1   : ISS(read);
 6955   src2   : ISS(read);
 6956   INS01  : ISS;
 6957   MAC    : WR;
 6958 %}
 6959 
 6960 // Multiply accumulate
 6961 // Eg.  MADD    w0, w1, w2, w3
 6962 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6963 %{
 6964   single_instruction;
 6965   dst    : WR(write);
 6966   src1   : ISS(read);
 6967   src2   : ISS(read);
 6968   src3   : ISS(read);
 6969   INS01  : ISS;
 6970   MAC    : WR;
 6971 %}
 6972 
 6973 // Eg.  MUL     w0, w1, w2
 6974 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6975 %{
 6976   single_instruction;
 6977   fixed_latency(3); // Maximum latency for 64 bit mul
 6978   dst    : WR(write);
 6979   src1   : ISS(read);
 6980   src2   : ISS(read);
 6981   INS01  : ISS;
 6982   MAC    : WR;
 6983 %}
 6984 
 6985 // Multiply accumulate
 6986 // Eg.  MADD    w0, w1, w2, w3
 6987 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6988 %{
 6989   single_instruction;
 6990   fixed_latency(3); // Maximum latency for 64 bit mul
 6991   dst    : WR(write);
 6992   src1   : ISS(read);
 6993   src2   : ISS(read);
 6994   src3   : ISS(read);
 6995   INS01  : ISS;
 6996   MAC    : WR;
 6997 %}
 6998 
 6999 //------- Divide pipeline operations --------------------
 7000 
 7001 // Eg.  SDIV    w0, w1, w2
 7002 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7003 %{
 7004   single_instruction;
 7005   fixed_latency(8); // Maximum latency for 32 bit divide
 7006   dst    : WR(write);
 7007   src1   : ISS(read);
 7008   src2   : ISS(read);
 7009   INS0   : ISS; // Can only dual issue as instruction 0
 7010   DIV    : WR;
 7011 %}
 7012 
 7013 // Eg.  SDIV    x0, x1, x2
 7014 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7015 %{
 7016   single_instruction;
 7017   fixed_latency(16); // Maximum latency for 64 bit divide
 7018   dst    : WR(write);
 7019   src1   : ISS(read);
 7020   src2   : ISS(read);
 7021   INS0   : ISS; // Can only dual issue as instruction 0
 7022   DIV    : WR;
 7023 %}
 7024 
 7025 //------- Load pipeline operations ------------------------
 7026 
 7027 // Load - prefetch
 7028 // Eg.  PFRM    <mem>
 7029 pipe_class iload_prefetch(memory mem)
 7030 %{
 7031   single_instruction;
 7032   mem    : ISS(read);
 7033   INS01  : ISS;
 7034   LDST   : WR;
 7035 %}
 7036 
 7037 // Load - reg, mem
 7038 // Eg.  LDR     x0, <mem>
 7039 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7040 %{
 7041   single_instruction;
 7042   dst    : WR(write);
 7043   mem    : ISS(read);
 7044   INS01  : ISS;
 7045   LDST   : WR;
 7046 %}
 7047 
 7048 // Load - reg, reg
 7049 // Eg.  LDR     x0, [sp, x1]
 7050 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7051 %{
 7052   single_instruction;
 7053   dst    : WR(write);
 7054   src    : ISS(read);
 7055   INS01  : ISS;
 7056   LDST   : WR;
 7057 %}
 7058 
 7059 //------- Store pipeline operations -----------------------
 7060 
 7061 // Store - zr, mem
 7062 // Eg.  STR     zr, <mem>
 7063 pipe_class istore_mem(memory mem)
 7064 %{
 7065   single_instruction;
 7066   mem    : ISS(read);
 7067   INS01  : ISS;
 7068   LDST   : WR;
 7069 %}
 7070 
 7071 // Store - reg, mem
 7072 // Eg.  STR     x0, <mem>
 7073 pipe_class istore_reg_mem(iRegI src, memory mem)
 7074 %{
 7075   single_instruction;
 7076   mem    : ISS(read);
 7077   src    : EX2(read);
 7078   INS01  : ISS;
 7079   LDST   : WR;
 7080 %}
 7081 
 7082 // Store - reg, reg
 7083 // Eg. STR      x0, [sp, x1]
 7084 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7085 %{
 7086   single_instruction;
 7087   dst    : ISS(read);
 7088   src    : EX2(read);
 7089   INS01  : ISS;
 7090   LDST   : WR;
 7091 %}
 7092 
 7093 //------- Store pipeline operations -----------------------
 7094 
 7095 // Branch
 7096 pipe_class pipe_branch()
 7097 %{
 7098   single_instruction;
 7099   INS01  : ISS;
 7100   BRANCH : EX1;
 7101 %}
 7102 
 7103 // Conditional branch
 7104 pipe_class pipe_branch_cond(rFlagsReg cr)
 7105 %{
 7106   single_instruction;
 7107   cr     : EX1(read);
 7108   INS01  : ISS;
 7109   BRANCH : EX1;
 7110 %}
 7111 
 7112 // Compare & Branch
 7113 // EG.  CBZ/CBNZ
 7114 pipe_class pipe_cmp_branch(iRegI op1)
 7115 %{
 7116   single_instruction;
 7117   op1    : EX1(read);
 7118   INS01  : ISS;
 7119   BRANCH : EX1;
 7120 %}
 7121 
 7122 //------- Synchronisation operations ----------------------
 7123 
 7124 // Any operation requiring serialization.
 7125 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7126 pipe_class pipe_serial()
 7127 %{
 7128   single_instruction;
 7129   force_serialization;
 7130   fixed_latency(16);
 7131   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7132   LDST   : WR;
 7133 %}
 7134 
 7135 // Generic big/slow expanded idiom - also serialized
 7136 pipe_class pipe_slow()
 7137 %{
 7138   instruction_count(10);
 7139   multiple_bundles;
 7140   force_serialization;
 7141   fixed_latency(16);
 7142   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7143   LDST   : WR;
 7144 %}
 7145 
 7146 // Empty pipeline class
 7147 pipe_class pipe_class_empty()
 7148 %{
 7149   single_instruction;
 7150   fixed_latency(0);
 7151 %}
 7152 
 7153 // Default pipeline class.
 7154 pipe_class pipe_class_default()
 7155 %{
 7156   single_instruction;
 7157   fixed_latency(2);
 7158 %}
 7159 
 7160 // Pipeline class for compares.
 7161 pipe_class pipe_class_compare()
 7162 %{
 7163   single_instruction;
 7164   fixed_latency(16);
 7165 %}
 7166 
 7167 // Pipeline class for memory operations.
 7168 pipe_class pipe_class_memory()
 7169 %{
 7170   single_instruction;
 7171   fixed_latency(16);
 7172 %}
 7173 
 7174 // Pipeline class for call.
 7175 pipe_class pipe_class_call()
 7176 %{
 7177   single_instruction;
 7178   fixed_latency(100);
 7179 %}
 7180 
 7181 // Define the class for the Nop node.
 7182 define %{
 7183    MachNop = pipe_class_empty;
 7184 %}
 7185 
 7186 %}
 7187 //----------INSTRUCTIONS-------------------------------------------------------
 7188 //
 7189 // match      -- States which machine-independent subtree may be replaced
 7190 //               by this instruction.
 7191 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7192 //               selection to identify a minimum cost tree of machine
 7193 //               instructions that matches a tree of machine-independent
 7194 //               instructions.
 7195 // format     -- A string providing the disassembly for this instruction.
 7196 //               The value of an instruction's operand may be inserted
 7197 //               by referring to it with a '$' prefix.
 7198 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7199 //               to within an encode class as $primary, $secondary, and $tertiary
 7200 //               rrspectively.  The primary opcode is commonly used to
 7201 //               indicate the type of machine instruction, while secondary
 7202 //               and tertiary are often used for prefix options or addressing
 7203 //               modes.
 7204 // ins_encode -- A list of encode classes with parameters. The encode class
 7205 //               name must have been defined in an 'enc_class' specification
 7206 //               in the encode section of the architecture description.
 7207 
 7208 // ============================================================================
 7209 // Memory (Load/Store) Instructions
 7210 
 7211 // Load Instructions
 7212 
 7213 // Load Byte (8 bit signed)
 7214 instruct loadB(iRegINoSp dst, memory1 mem)
 7215 %{
 7216   match(Set dst (LoadB mem));
 7217   predicate(!needs_acquiring_load(n));
 7218 
 7219   ins_cost(4 * INSN_COST);
 7220   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7221 
 7222   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7223 
 7224   ins_pipe(iload_reg_mem);
 7225 %}
 7226 
 7227 // Load Byte (8 bit signed) into long
 7228 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7229 %{
 7230   match(Set dst (ConvI2L (LoadB mem)));
 7231   predicate(!needs_acquiring_load(n->in(1)));
 7232 
 7233   ins_cost(4 * INSN_COST);
 7234   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7235 
 7236   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7237 
 7238   ins_pipe(iload_reg_mem);
 7239 %}
 7240 
 7241 // Load Byte (8 bit unsigned)
 7242 instruct loadUB(iRegINoSp dst, memory1 mem)
 7243 %{
 7244   match(Set dst (LoadUB mem));
 7245   predicate(!needs_acquiring_load(n));
 7246 
 7247   ins_cost(4 * INSN_COST);
 7248   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7249 
 7250   ins_encode(aarch64_enc_ldrb(dst, mem));
 7251 
 7252   ins_pipe(iload_reg_mem);
 7253 %}
 7254 
 7255 // Load Byte (8 bit unsigned) into long
 7256 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7257 %{
 7258   match(Set dst (ConvI2L (LoadUB mem)));
 7259   predicate(!needs_acquiring_load(n->in(1)));
 7260 
 7261   ins_cost(4 * INSN_COST);
 7262   format %{ "ldrb  $dst, $mem\t# byte" %}
 7263 
 7264   ins_encode(aarch64_enc_ldrb(dst, mem));
 7265 
 7266   ins_pipe(iload_reg_mem);
 7267 %}
 7268 
 7269 // Load Short (16 bit signed)
 7270 instruct loadS(iRegINoSp dst, memory2 mem)
 7271 %{
 7272   match(Set dst (LoadS mem));
 7273   predicate(!needs_acquiring_load(n));
 7274 
 7275   ins_cost(4 * INSN_COST);
 7276   format %{ "ldrshw  $dst, $mem\t# short" %}
 7277 
 7278   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7279 
 7280   ins_pipe(iload_reg_mem);
 7281 %}
 7282 
 7283 // Load Short (16 bit signed) into long
 7284 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7285 %{
 7286   match(Set dst (ConvI2L (LoadS mem)));
 7287   predicate(!needs_acquiring_load(n->in(1)));
 7288 
 7289   ins_cost(4 * INSN_COST);
 7290   format %{ "ldrsh  $dst, $mem\t# short" %}
 7291 
 7292   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7293 
 7294   ins_pipe(iload_reg_mem);
 7295 %}
 7296 
 7297 // Load Char (16 bit unsigned)
 7298 instruct loadUS(iRegINoSp dst, memory2 mem)
 7299 %{
 7300   match(Set dst (LoadUS mem));
 7301   predicate(!needs_acquiring_load(n));
 7302 
 7303   ins_cost(4 * INSN_COST);
 7304   format %{ "ldrh  $dst, $mem\t# short" %}
 7305 
 7306   ins_encode(aarch64_enc_ldrh(dst, mem));
 7307 
 7308   ins_pipe(iload_reg_mem);
 7309 %}
 7310 
 7311 // Load Short/Char (16 bit unsigned) into long
 7312 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7313 %{
 7314   match(Set dst (ConvI2L (LoadUS mem)));
 7315   predicate(!needs_acquiring_load(n->in(1)));
 7316 
 7317   ins_cost(4 * INSN_COST);
 7318   format %{ "ldrh  $dst, $mem\t# short" %}
 7319 
 7320   ins_encode(aarch64_enc_ldrh(dst, mem));
 7321 
 7322   ins_pipe(iload_reg_mem);
 7323 %}
 7324 
 7325 // Load Integer (32 bit signed)
 7326 instruct loadI(iRegINoSp dst, memory4 mem)
 7327 %{
 7328   match(Set dst (LoadI mem));
 7329   predicate(!needs_acquiring_load(n));
 7330 
 7331   ins_cost(4 * INSN_COST);
 7332   format %{ "ldrw  $dst, $mem\t# int" %}
 7333 
 7334   ins_encode(aarch64_enc_ldrw(dst, mem));
 7335 
 7336   ins_pipe(iload_reg_mem);
 7337 %}
 7338 
 7339 // Load Integer (32 bit signed) into long
 7340 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7341 %{
 7342   match(Set dst (ConvI2L (LoadI mem)));
 7343   predicate(!needs_acquiring_load(n->in(1)));
 7344 
 7345   ins_cost(4 * INSN_COST);
 7346   format %{ "ldrsw  $dst, $mem\t# int" %}
 7347 
 7348   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7349 
 7350   ins_pipe(iload_reg_mem);
 7351 %}
 7352 
 7353 // Load Integer (32 bit unsigned) into long
 7354 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7355 %{
 7356   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7357   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7358 
 7359   ins_cost(4 * INSN_COST);
 7360   format %{ "ldrw  $dst, $mem\t# int" %}
 7361 
 7362   ins_encode(aarch64_enc_ldrw(dst, mem));
 7363 
 7364   ins_pipe(iload_reg_mem);
 7365 %}
 7366 
 7367 // Load Long (64 bit signed)
 7368 instruct loadL(iRegLNoSp dst, memory8 mem)
 7369 %{
 7370   match(Set dst (LoadL mem));
 7371   predicate(!needs_acquiring_load(n));
 7372 
 7373   ins_cost(4 * INSN_COST);
 7374   format %{ "ldr  $dst, $mem\t# int" %}
 7375 
 7376   ins_encode(aarch64_enc_ldr(dst, mem));
 7377 
 7378   ins_pipe(iload_reg_mem);
 7379 %}
 7380 
 7381 // Load Range
 7382 instruct loadRange(iRegINoSp dst, memory4 mem)
 7383 %{
 7384   match(Set dst (LoadRange mem));
 7385 
 7386   ins_cost(4 * INSN_COST);
 7387   format %{ "ldrw  $dst, $mem\t# range" %}
 7388 
 7389   ins_encode(aarch64_enc_ldrw(dst, mem));
 7390 
 7391   ins_pipe(iload_reg_mem);
 7392 %}
 7393 
 7394 // Load Pointer
 7395 instruct loadP(iRegPNoSp dst, memory8 mem)
 7396 %{
 7397   match(Set dst (LoadP mem));
 7398   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7399 
 7400   ins_cost(4 * INSN_COST);
 7401   format %{ "ldr  $dst, $mem\t# ptr" %}
 7402 
 7403   ins_encode(aarch64_enc_ldr(dst, mem));
 7404 
 7405   ins_pipe(iload_reg_mem);
 7406 %}
 7407 
 7408 // Load Compressed Pointer
 7409 instruct loadN(iRegNNoSp dst, memory4 mem)
 7410 %{
 7411   match(Set dst (LoadN mem));
 7412   predicate(!needs_acquiring_load(n));
 7413 
 7414   ins_cost(4 * INSN_COST);
 7415   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7416 
 7417   ins_encode(aarch64_enc_ldrw(dst, mem));
 7418 
 7419   ins_pipe(iload_reg_mem);
 7420 %}
 7421 
 7422 // Load Klass Pointer
 7423 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7424 %{
 7425   match(Set dst (LoadKlass mem));
 7426   predicate(!needs_acquiring_load(n));
 7427 
 7428   ins_cost(4 * INSN_COST);
 7429   format %{ "ldr  $dst, $mem\t# class" %}
 7430 
 7431   ins_encode(aarch64_enc_ldr(dst, mem));
 7432 
 7433   ins_pipe(iload_reg_mem);
 7434 %}
 7435 
 7436 // Load Narrow Klass Pointer
 7437 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7438 %{
 7439   match(Set dst (LoadNKlass mem));
 7440   predicate(!needs_acquiring_load(n));
 7441 
 7442   ins_cost(4 * INSN_COST);
 7443   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7444 
 7445   ins_encode(aarch64_enc_ldrw(dst, mem));
 7446 
 7447   ins_pipe(iload_reg_mem);
 7448 %}
 7449 
 7450 // Load Float
 7451 instruct loadF(vRegF dst, memory4 mem)
 7452 %{
 7453   match(Set dst (LoadF mem));
 7454   predicate(!needs_acquiring_load(n));
 7455 
 7456   ins_cost(4 * INSN_COST);
 7457   format %{ "ldrs  $dst, $mem\t# float" %}
 7458 
 7459   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7460 
 7461   ins_pipe(pipe_class_memory);
 7462 %}
 7463 
 7464 // Load Double
 7465 instruct loadD(vRegD dst, memory8 mem)
 7466 %{
 7467   match(Set dst (LoadD mem));
 7468   predicate(!needs_acquiring_load(n));
 7469 
 7470   ins_cost(4 * INSN_COST);
 7471   format %{ "ldrd  $dst, $mem\t# double" %}
 7472 
 7473   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7474 
 7475   ins_pipe(pipe_class_memory);
 7476 %}
 7477 
 7478 
 7479 // Load Int Constant
 7480 instruct loadConI(iRegINoSp dst, immI src)
 7481 %{
 7482   match(Set dst src);
 7483 
 7484   ins_cost(INSN_COST);
 7485   format %{ "mov $dst, $src\t# int" %}
 7486 
 7487   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7488 
 7489   ins_pipe(ialu_imm);
 7490 %}
 7491 
 7492 // Load Long Constant
 7493 instruct loadConL(iRegLNoSp dst, immL src)
 7494 %{
 7495   match(Set dst src);
 7496 
 7497   ins_cost(INSN_COST);
 7498   format %{ "mov $dst, $src\t# long" %}
 7499 
 7500   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7501 
 7502   ins_pipe(ialu_imm);
 7503 %}
 7504 
 7505 // Load Pointer Constant
 7506 
 7507 instruct loadConP(iRegPNoSp dst, immP con)
 7508 %{
 7509   match(Set dst con);
 7510 
 7511   ins_cost(INSN_COST * 4);
 7512   format %{
 7513     "mov  $dst, $con\t# ptr\n\t"
 7514   %}
 7515 
 7516   ins_encode(aarch64_enc_mov_p(dst, con));
 7517 
 7518   ins_pipe(ialu_imm);
 7519 %}
 7520 
 7521 // Load Null Pointer Constant
 7522 
 7523 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7524 %{
 7525   match(Set dst con);
 7526 
 7527   ins_cost(INSN_COST);
 7528   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7529 
 7530   ins_encode(aarch64_enc_mov_p0(dst, con));
 7531 
 7532   ins_pipe(ialu_imm);
 7533 %}
 7534 
 7535 // Load Pointer Constant One
 7536 
 7537 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7538 %{
 7539   match(Set dst con);
 7540 
 7541   ins_cost(INSN_COST);
 7542   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7543 
 7544   ins_encode(aarch64_enc_mov_p1(dst, con));
 7545 
 7546   ins_pipe(ialu_imm);
 7547 %}
 7548 
 7549 // Load Byte Map Base Constant
 7550 
 7551 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7552 %{
 7553   match(Set dst con);
 7554 
 7555   ins_cost(INSN_COST);
 7556   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7557 
 7558   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7559 
 7560   ins_pipe(ialu_imm);
 7561 %}
 7562 
 7563 // Load Narrow Pointer Constant
 7564 
 7565 instruct loadConN(iRegNNoSp dst, immN con)
 7566 %{
 7567   match(Set dst con);
 7568 
 7569   ins_cost(INSN_COST * 4);
 7570   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7571 
 7572   ins_encode(aarch64_enc_mov_n(dst, con));
 7573 
 7574   ins_pipe(ialu_imm);
 7575 %}
 7576 
 7577 // Load Narrow Null Pointer Constant
 7578 
 7579 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7580 %{
 7581   match(Set dst con);
 7582 
 7583   ins_cost(INSN_COST);
 7584   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7585 
 7586   ins_encode(aarch64_enc_mov_n0(dst, con));
 7587 
 7588   ins_pipe(ialu_imm);
 7589 %}
 7590 
 7591 // Load Narrow Klass Constant
 7592 
 7593 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7594 %{
 7595   match(Set dst con);
 7596 
 7597   ins_cost(INSN_COST);
 7598   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7599 
 7600   ins_encode(aarch64_enc_mov_nk(dst, con));
 7601 
 7602   ins_pipe(ialu_imm);
 7603 %}
 7604 
 7605 // Load Packed Float Constant
 7606 
 7607 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7608   match(Set dst con);
 7609   ins_cost(INSN_COST * 4);
 7610   format %{ "fmovs  $dst, $con"%}
 7611   ins_encode %{
 7612     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7613   %}
 7614 
 7615   ins_pipe(fp_imm_s);
 7616 %}
 7617 
 7618 // Load Float Constant
 7619 
 7620 instruct loadConF(vRegF dst, immF con) %{
 7621   match(Set dst con);
 7622 
 7623   ins_cost(INSN_COST * 4);
 7624 
 7625   format %{
 7626     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7627   %}
 7628 
 7629   ins_encode %{
 7630     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7631   %}
 7632 
 7633   ins_pipe(fp_load_constant_s);
 7634 %}
 7635 
 7636 // Load Packed Double Constant
 7637 
 7638 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7639   match(Set dst con);
 7640   ins_cost(INSN_COST);
 7641   format %{ "fmovd  $dst, $con"%}
 7642   ins_encode %{
 7643     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7644   %}
 7645 
 7646   ins_pipe(fp_imm_d);
 7647 %}
 7648 
 7649 // Load Double Constant
 7650 
 7651 instruct loadConD(vRegD dst, immD con) %{
 7652   match(Set dst con);
 7653 
 7654   ins_cost(INSN_COST * 5);
 7655   format %{
 7656     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7657   %}
 7658 
 7659   ins_encode %{
 7660     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7661   %}
 7662 
 7663   ins_pipe(fp_load_constant_d);
 7664 %}
 7665 
 7666 // Store Instructions
 7667 
 7668 // Store CMS card-mark Immediate
 7669 instruct storeimmCM0(immI0 zero, memory1 mem)
 7670 %{
 7671   match(Set mem (StoreCM mem zero));
 7672 
 7673   ins_cost(INSN_COST);
 7674   format %{ "storestore (elided)\n\t"
 7675             "strb zr, $mem\t# byte" %}
 7676 
 7677   ins_encode(aarch64_enc_strb0(mem));
 7678 
 7679   ins_pipe(istore_mem);
 7680 %}
 7681 
 7682 // Store CMS card-mark Immediate with intervening StoreStore
 7683 // needed when using CMS with no conditional card marking
 7684 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7685 %{
 7686   match(Set mem (StoreCM mem zero));
 7687 
 7688   ins_cost(INSN_COST * 2);
 7689   format %{ "storestore\n\t"
 7690             "dmb ishst"
 7691             "\n\tstrb zr, $mem\t# byte" %}
 7692 
 7693   ins_encode(aarch64_enc_strb0_ordered(mem));
 7694 
 7695   ins_pipe(istore_mem);
 7696 %}
 7697 
 7698 // Store Byte
 7699 instruct storeB(iRegIorL2I src, memory1 mem)
 7700 %{
 7701   match(Set mem (StoreB mem src));
 7702   predicate(!needs_releasing_store(n));
 7703 
 7704   ins_cost(INSN_COST);
 7705   format %{ "strb  $src, $mem\t# byte" %}
 7706 
 7707   ins_encode(aarch64_enc_strb(src, mem));
 7708 
 7709   ins_pipe(istore_reg_mem);
 7710 %}
 7711 
 7712 
 7713 instruct storeimmB0(immI0 zero, memory1 mem)
 7714 %{
 7715   match(Set mem (StoreB mem zero));
 7716   predicate(!needs_releasing_store(n));
 7717 
 7718   ins_cost(INSN_COST);
 7719   format %{ "strb rscractch2, $mem\t# byte" %}
 7720 
 7721   ins_encode(aarch64_enc_strb0(mem));
 7722 
 7723   ins_pipe(istore_mem);
 7724 %}
 7725 
 7726 // Store Char/Short
 7727 instruct storeC(iRegIorL2I src, memory2 mem)
 7728 %{
 7729   match(Set mem (StoreC mem src));
 7730   predicate(!needs_releasing_store(n));
 7731 
 7732   ins_cost(INSN_COST);
 7733   format %{ "strh  $src, $mem\t# short" %}
 7734 
 7735   ins_encode(aarch64_enc_strh(src, mem));
 7736 
 7737   ins_pipe(istore_reg_mem);
 7738 %}
 7739 
 7740 instruct storeimmC0(immI0 zero, memory2 mem)
 7741 %{
 7742   match(Set mem (StoreC mem zero));
 7743   predicate(!needs_releasing_store(n));
 7744 
 7745   ins_cost(INSN_COST);
 7746   format %{ "strh  zr, $mem\t# short" %}
 7747 
 7748   ins_encode(aarch64_enc_strh0(mem));
 7749 
 7750   ins_pipe(istore_mem);
 7751 %}
 7752 
 7753 // Store Integer
 7754 
 7755 instruct storeI(iRegIorL2I src, memory4 mem)
 7756 %{
 7757   match(Set mem(StoreI mem src));
 7758   predicate(!needs_releasing_store(n));
 7759 
 7760   ins_cost(INSN_COST);
 7761   format %{ "strw  $src, $mem\t# int" %}
 7762 
 7763   ins_encode(aarch64_enc_strw(src, mem));
 7764 
 7765   ins_pipe(istore_reg_mem);
 7766 %}
 7767 
 7768 instruct storeimmI0(immI0 zero, memory4 mem)
 7769 %{
 7770   match(Set mem(StoreI mem zero));
 7771   predicate(!needs_releasing_store(n));
 7772 
 7773   ins_cost(INSN_COST);
 7774   format %{ "strw  zr, $mem\t# int" %}
 7775 
 7776   ins_encode(aarch64_enc_strw0(mem));
 7777 
 7778   ins_pipe(istore_mem);
 7779 %}
 7780 
 7781 // Store Long (64 bit signed)
 7782 instruct storeL(iRegL src, memory8 mem)
 7783 %{
 7784   match(Set mem (StoreL mem src));
 7785   predicate(!needs_releasing_store(n));
 7786 
 7787   ins_cost(INSN_COST);
 7788   format %{ "str  $src, $mem\t# int" %}
 7789 
 7790   ins_encode(aarch64_enc_str(src, mem));
 7791 
 7792   ins_pipe(istore_reg_mem);
 7793 %}
 7794 
 7795 // Store Long (64 bit signed)
 7796 instruct storeimmL0(immL0 zero, memory8 mem)
 7797 %{
 7798   match(Set mem (StoreL mem zero));
 7799   predicate(!needs_releasing_store(n));
 7800 
 7801   ins_cost(INSN_COST);
 7802   format %{ "str  zr, $mem\t# int" %}
 7803 
 7804   ins_encode(aarch64_enc_str0(mem));
 7805 
 7806   ins_pipe(istore_mem);
 7807 %}
 7808 
 7809 // Store Pointer
 7810 instruct storeP(iRegP src, memory8 mem)
 7811 %{
 7812   match(Set mem (StoreP mem src));
 7813   predicate(!needs_releasing_store(n));
 7814 
 7815   ins_cost(INSN_COST);
 7816   format %{ "str  $src, $mem\t# ptr" %}
 7817 
 7818   ins_encode(aarch64_enc_str(src, mem));
 7819 
 7820   ins_pipe(istore_reg_mem);
 7821 %}
 7822 
 7823 // Store Pointer
 7824 instruct storeimmP0(immP0 zero, memory8 mem)
 7825 %{
 7826   match(Set mem (StoreP mem zero));
 7827   predicate(!needs_releasing_store(n));
 7828 
 7829   ins_cost(INSN_COST);
 7830   format %{ "str zr, $mem\t# ptr" %}
 7831 
 7832   ins_encode(aarch64_enc_str0(mem));
 7833 
 7834   ins_pipe(istore_mem);
 7835 %}
 7836 
 7837 // Store Compressed Pointer
 7838 instruct storeN(iRegN src, memory4 mem)
 7839 %{
 7840   match(Set mem (StoreN mem src));
 7841   predicate(!needs_releasing_store(n));
 7842 
 7843   ins_cost(INSN_COST);
 7844   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7845 
 7846   ins_encode(aarch64_enc_strw(src, mem));
 7847 
 7848   ins_pipe(istore_reg_mem);
 7849 %}
 7850 
 7851 instruct storeImmN0(immN0 zero, memory4 mem)
 7852 %{
 7853   match(Set mem (StoreN mem zero));
 7854   predicate(!needs_releasing_store(n));
 7855 
 7856   ins_cost(INSN_COST);
 7857   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7858 
 7859   ins_encode(aarch64_enc_strw0(mem));
 7860 
 7861   ins_pipe(istore_mem);
 7862 %}
 7863 
 7864 // Store Float
 7865 instruct storeF(vRegF src, memory4 mem)
 7866 %{
 7867   match(Set mem (StoreF mem src));
 7868   predicate(!needs_releasing_store(n));
 7869 
 7870   ins_cost(INSN_COST);
 7871   format %{ "strs  $src, $mem\t# float" %}
 7872 
 7873   ins_encode( aarch64_enc_strs(src, mem) );
 7874 
 7875   ins_pipe(pipe_class_memory);
 7876 %}
 7877 
 7878 // TODO
 7879 // implement storeImmF0 and storeFImmPacked
 7880 
 7881 // Store Double
 7882 instruct storeD(vRegD src, memory8 mem)
 7883 %{
 7884   match(Set mem (StoreD mem src));
 7885   predicate(!needs_releasing_store(n));
 7886 
 7887   ins_cost(INSN_COST);
 7888   format %{ "strd  $src, $mem\t# double" %}
 7889 
 7890   ins_encode( aarch64_enc_strd(src, mem) );
 7891 
 7892   ins_pipe(pipe_class_memory);
 7893 %}
 7894 
 7895 // Store Compressed Klass Pointer
 7896 instruct storeNKlass(iRegN src, memory4 mem)
 7897 %{
 7898   predicate(!needs_releasing_store(n));
 7899   match(Set mem (StoreNKlass mem src));
 7900 
 7901   ins_cost(INSN_COST);
 7902   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7903 
 7904   ins_encode(aarch64_enc_strw(src, mem));
 7905 
 7906   ins_pipe(istore_reg_mem);
 7907 %}
 7908 
 7909 // TODO
 7910 // implement storeImmD0 and storeDImmPacked
 7911 
 7912 // prefetch instructions
 7913 // Must be safe to execute with invalid address (cannot fault).
 7914 
 7915 instruct prefetchalloc( memory8 mem ) %{
 7916   match(PrefetchAllocation mem);
 7917 
 7918   ins_cost(INSN_COST);
 7919   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7920 
 7921   ins_encode( aarch64_enc_prefetchw(mem) );
 7922 
 7923   ins_pipe(iload_prefetch);
 7924 %}
 7925 
 7926 //  ---------------- volatile loads and stores ----------------
 7927 
 7928 // Load Byte (8 bit signed)
 7929 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7930 %{
 7931   match(Set dst (LoadB mem));
 7932 
 7933   ins_cost(VOLATILE_REF_COST);
 7934   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7935 
 7936   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7937 
 7938   ins_pipe(pipe_serial);
 7939 %}
 7940 
 7941 // Load Byte (8 bit signed) into long
 7942 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7943 %{
 7944   match(Set dst (ConvI2L (LoadB mem)));
 7945 
 7946   ins_cost(VOLATILE_REF_COST);
 7947   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7948 
 7949   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7950 
 7951   ins_pipe(pipe_serial);
 7952 %}
 7953 
 7954 // Load Byte (8 bit unsigned)
 7955 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7956 %{
 7957   match(Set dst (LoadUB mem));
 7958 
 7959   ins_cost(VOLATILE_REF_COST);
 7960   format %{ "ldarb  $dst, $mem\t# byte" %}
 7961 
 7962   ins_encode(aarch64_enc_ldarb(dst, mem));
 7963 
 7964   ins_pipe(pipe_serial);
 7965 %}
 7966 
 7967 // Load Byte (8 bit unsigned) into long
 7968 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7969 %{
 7970   match(Set dst (ConvI2L (LoadUB mem)));
 7971 
 7972   ins_cost(VOLATILE_REF_COST);
 7973   format %{ "ldarb  $dst, $mem\t# byte" %}
 7974 
 7975   ins_encode(aarch64_enc_ldarb(dst, mem));
 7976 
 7977   ins_pipe(pipe_serial);
 7978 %}
 7979 
 7980 // Load Short (16 bit signed)
 7981 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7982 %{
 7983   match(Set dst (LoadS mem));
 7984 
 7985   ins_cost(VOLATILE_REF_COST);
 7986   format %{ "ldarshw  $dst, $mem\t# short" %}
 7987 
 7988   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7989 
 7990   ins_pipe(pipe_serial);
 7991 %}
 7992 
 7993 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7994 %{
 7995   match(Set dst (LoadUS mem));
 7996 
 7997   ins_cost(VOLATILE_REF_COST);
 7998   format %{ "ldarhw  $dst, $mem\t# short" %}
 7999 
 8000   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8001 
 8002   ins_pipe(pipe_serial);
 8003 %}
 8004 
 8005 // Load Short/Char (16 bit unsigned) into long
 8006 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8007 %{
 8008   match(Set dst (ConvI2L (LoadUS mem)));
 8009 
 8010   ins_cost(VOLATILE_REF_COST);
 8011   format %{ "ldarh  $dst, $mem\t# short" %}
 8012 
 8013   ins_encode(aarch64_enc_ldarh(dst, mem));
 8014 
 8015   ins_pipe(pipe_serial);
 8016 %}
 8017 
 8018 // Load Short/Char (16 bit signed) into long
 8019 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8020 %{
 8021   match(Set dst (ConvI2L (LoadS mem)));
 8022 
 8023   ins_cost(VOLATILE_REF_COST);
 8024   format %{ "ldarh  $dst, $mem\t# short" %}
 8025 
 8026   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8027 
 8028   ins_pipe(pipe_serial);
 8029 %}
 8030 
 8031 // Load Integer (32 bit signed)
 8032 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8033 %{
 8034   match(Set dst (LoadI mem));
 8035 
 8036   ins_cost(VOLATILE_REF_COST);
 8037   format %{ "ldarw  $dst, $mem\t# int" %}
 8038 
 8039   ins_encode(aarch64_enc_ldarw(dst, mem));
 8040 
 8041   ins_pipe(pipe_serial);
 8042 %}
 8043 
 8044 // Load Integer (32 bit unsigned) into long
 8045 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8046 %{
 8047   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8048 
 8049   ins_cost(VOLATILE_REF_COST);
 8050   format %{ "ldarw  $dst, $mem\t# int" %}
 8051 
 8052   ins_encode(aarch64_enc_ldarw(dst, mem));
 8053 
 8054   ins_pipe(pipe_serial);
 8055 %}
 8056 
 8057 // Load Long (64 bit signed)
 8058 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8059 %{
 8060   match(Set dst (LoadL mem));
 8061 
 8062   ins_cost(VOLATILE_REF_COST);
 8063   format %{ "ldar  $dst, $mem\t# int" %}
 8064 
 8065   ins_encode(aarch64_enc_ldar(dst, mem));
 8066 
 8067   ins_pipe(pipe_serial);
 8068 %}
 8069 
 8070 // Load Pointer
 8071 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8072 %{
 8073   match(Set dst (LoadP mem));
 8074   predicate(n->as_Load()->barrier_data() == 0);
 8075 
 8076   ins_cost(VOLATILE_REF_COST);
 8077   format %{ "ldar  $dst, $mem\t# ptr" %}
 8078 
 8079   ins_encode(aarch64_enc_ldar(dst, mem));
 8080 
 8081   ins_pipe(pipe_serial);
 8082 %}
 8083 
 8084 // Load Compressed Pointer
 8085 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8086 %{
 8087   match(Set dst (LoadN mem));
 8088 
 8089   ins_cost(VOLATILE_REF_COST);
 8090   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8091 
 8092   ins_encode(aarch64_enc_ldarw(dst, mem));
 8093 
 8094   ins_pipe(pipe_serial);
 8095 %}
 8096 
 8097 // Load Float
 8098 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8099 %{
 8100   match(Set dst (LoadF mem));
 8101 
 8102   ins_cost(VOLATILE_REF_COST);
 8103   format %{ "ldars  $dst, $mem\t# float" %}
 8104 
 8105   ins_encode( aarch64_enc_fldars(dst, mem) );
 8106 
 8107   ins_pipe(pipe_serial);
 8108 %}
 8109 
 8110 // Load Double
 8111 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8112 %{
 8113   match(Set dst (LoadD mem));
 8114 
 8115   ins_cost(VOLATILE_REF_COST);
 8116   format %{ "ldard  $dst, $mem\t# double" %}
 8117 
 8118   ins_encode( aarch64_enc_fldard(dst, mem) );
 8119 
 8120   ins_pipe(pipe_serial);
 8121 %}
 8122 
 8123 // Store Byte
 8124 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8125 %{
 8126   match(Set mem (StoreB mem src));
 8127 
 8128   ins_cost(VOLATILE_REF_COST);
 8129   format %{ "stlrb  $src, $mem\t# byte" %}
 8130 
 8131   ins_encode(aarch64_enc_stlrb(src, mem));
 8132 
 8133   ins_pipe(pipe_class_memory);
 8134 %}
 8135 
 8136 // Store Char/Short
 8137 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8138 %{
 8139   match(Set mem (StoreC mem src));
 8140 
 8141   ins_cost(VOLATILE_REF_COST);
 8142   format %{ "stlrh  $src, $mem\t# short" %}
 8143 
 8144   ins_encode(aarch64_enc_stlrh(src, mem));
 8145 
 8146   ins_pipe(pipe_class_memory);
 8147 %}
 8148 
 8149 // Store Integer
 8150 
 8151 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8152 %{
 8153   match(Set mem(StoreI mem src));
 8154 
 8155   ins_cost(VOLATILE_REF_COST);
 8156   format %{ "stlrw  $src, $mem\t# int" %}
 8157 
 8158   ins_encode(aarch64_enc_stlrw(src, mem));
 8159 
 8160   ins_pipe(pipe_class_memory);
 8161 %}
 8162 
 8163 // Store Long (64 bit signed)
 8164 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8165 %{
 8166   match(Set mem (StoreL mem src));
 8167 
 8168   ins_cost(VOLATILE_REF_COST);
 8169   format %{ "stlr  $src, $mem\t# int" %}
 8170 
 8171   ins_encode(aarch64_enc_stlr(src, mem));
 8172 
 8173   ins_pipe(pipe_class_memory);
 8174 %}
 8175 
 8176 // Store Pointer
 8177 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8178 %{
 8179   match(Set mem (StoreP mem src));
 8180 
 8181   ins_cost(VOLATILE_REF_COST);
 8182   format %{ "stlr  $src, $mem\t# ptr" %}
 8183 
 8184   ins_encode(aarch64_enc_stlr(src, mem));
 8185 
 8186   ins_pipe(pipe_class_memory);
 8187 %}
 8188 
 8189 // Store Compressed Pointer
 8190 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8191 %{
 8192   match(Set mem (StoreN mem src));
 8193 
 8194   ins_cost(VOLATILE_REF_COST);
 8195   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8196 
 8197   ins_encode(aarch64_enc_stlrw(src, mem));
 8198 
 8199   ins_pipe(pipe_class_memory);
 8200 %}
 8201 
 8202 // Store Float
 8203 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8204 %{
 8205   match(Set mem (StoreF mem src));
 8206 
 8207   ins_cost(VOLATILE_REF_COST);
 8208   format %{ "stlrs  $src, $mem\t# float" %}
 8209 
 8210   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8211 
 8212   ins_pipe(pipe_class_memory);
 8213 %}
 8214 
 8215 // TODO
 8216 // implement storeImmF0 and storeFImmPacked
 8217 
 8218 // Store Double
 8219 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8220 %{
 8221   match(Set mem (StoreD mem src));
 8222 
 8223   ins_cost(VOLATILE_REF_COST);
 8224   format %{ "stlrd  $src, $mem\t# double" %}
 8225 
 8226   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8227 
 8228   ins_pipe(pipe_class_memory);
 8229 %}
 8230 
 8231 //  ---------------- end of volatile loads and stores ----------------
 8232 
 8233 instruct cacheWB(indirect addr)
 8234 %{
 8235   predicate(VM_Version::supports_data_cache_line_flush());
 8236   match(CacheWB addr);
 8237 
 8238   ins_cost(100);
 8239   format %{"cache wb $addr" %}
 8240   ins_encode %{
 8241     assert($addr->index_position() < 0, "should be");
 8242     assert($addr$$disp == 0, "should be");
 8243     __ cache_wb(Address($addr$$base$$Register, 0));
 8244   %}
 8245   ins_pipe(pipe_slow); // XXX
 8246 %}
 8247 
 8248 instruct cacheWBPreSync()
 8249 %{
 8250   predicate(VM_Version::supports_data_cache_line_flush());
 8251   match(CacheWBPreSync);
 8252 
 8253   ins_cost(100);
 8254   format %{"cache wb presync" %}
 8255   ins_encode %{
 8256     __ cache_wbsync(true);
 8257   %}
 8258   ins_pipe(pipe_slow); // XXX
 8259 %}
 8260 
 8261 instruct cacheWBPostSync()
 8262 %{
 8263   predicate(VM_Version::supports_data_cache_line_flush());
 8264   match(CacheWBPostSync);
 8265 
 8266   ins_cost(100);
 8267   format %{"cache wb postsync" %}
 8268   ins_encode %{
 8269     __ cache_wbsync(false);
 8270   %}
 8271   ins_pipe(pipe_slow); // XXX
 8272 %}
 8273 
 8274 // ============================================================================
 8275 // BSWAP Instructions
 8276 
 8277 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8278   match(Set dst (ReverseBytesI src));
 8279 
 8280   ins_cost(INSN_COST);
 8281   format %{ "revw  $dst, $src" %}
 8282 
 8283   ins_encode %{
 8284     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8285   %}
 8286 
 8287   ins_pipe(ialu_reg);
 8288 %}
 8289 
 8290 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8291   match(Set dst (ReverseBytesL src));
 8292 
 8293   ins_cost(INSN_COST);
 8294   format %{ "rev  $dst, $src" %}
 8295 
 8296   ins_encode %{
 8297     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8298   %}
 8299 
 8300   ins_pipe(ialu_reg);
 8301 %}
 8302 
 8303 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8304   match(Set dst (ReverseBytesUS src));
 8305 
 8306   ins_cost(INSN_COST);
 8307   format %{ "rev16w  $dst, $src" %}
 8308 
 8309   ins_encode %{
 8310     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8311   %}
 8312 
 8313   ins_pipe(ialu_reg);
 8314 %}
 8315 
 8316 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8317   match(Set dst (ReverseBytesS src));
 8318 
 8319   ins_cost(INSN_COST);
 8320   format %{ "rev16w  $dst, $src\n\t"
 8321             "sbfmw $dst, $dst, #0, #15" %}
 8322 
 8323   ins_encode %{
 8324     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8325     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8326   %}
 8327 
 8328   ins_pipe(ialu_reg);
 8329 %}
 8330 
 8331 // ============================================================================
 8332 // Zero Count Instructions
 8333 
 8334 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8335   match(Set dst (CountLeadingZerosI src));
 8336 
 8337   ins_cost(INSN_COST);
 8338   format %{ "clzw  $dst, $src" %}
 8339   ins_encode %{
 8340     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8341   %}
 8342 
 8343   ins_pipe(ialu_reg);
 8344 %}
 8345 
 8346 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8347   match(Set dst (CountLeadingZerosL src));
 8348 
 8349   ins_cost(INSN_COST);
 8350   format %{ "clz   $dst, $src" %}
 8351   ins_encode %{
 8352     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8353   %}
 8354 
 8355   ins_pipe(ialu_reg);
 8356 %}
 8357 
 8358 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8359   match(Set dst (CountTrailingZerosI src));
 8360 
 8361   ins_cost(INSN_COST * 2);
 8362   format %{ "rbitw  $dst, $src\n\t"
 8363             "clzw   $dst, $dst" %}
 8364   ins_encode %{
 8365     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8366     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8367   %}
 8368 
 8369   ins_pipe(ialu_reg);
 8370 %}
 8371 
 8372 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8373   match(Set dst (CountTrailingZerosL src));
 8374 
 8375   ins_cost(INSN_COST * 2);
 8376   format %{ "rbit   $dst, $src\n\t"
 8377             "clz    $dst, $dst" %}
 8378   ins_encode %{
 8379     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8380     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8381   %}
 8382 
 8383   ins_pipe(ialu_reg);
 8384 %}
 8385 
 8386 //---------- Population Count Instructions -------------------------------------
 8387 //
 8388 
 8389 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8390   predicate(UsePopCountInstruction);
 8391   match(Set dst (PopCountI src));
 8392   effect(TEMP tmp);
 8393   ins_cost(INSN_COST * 13);
 8394 
 8395   format %{ "movw   $src, $src\n\t"
 8396             "mov    $tmp, $src\t# vector (1D)\n\t"
 8397             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8398             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8399             "mov    $dst, $tmp\t# vector (1D)" %}
 8400   ins_encode %{
 8401     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8402     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8403     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8404     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8405     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8406   %}
 8407 
 8408   ins_pipe(pipe_class_default);
 8409 %}
 8410 
 8411 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8412   predicate(UsePopCountInstruction);
 8413   match(Set dst (PopCountI (LoadI mem)));
 8414   effect(TEMP tmp);
 8415   ins_cost(INSN_COST * 13);
 8416 
 8417   format %{ "ldrs   $tmp, $mem\n\t"
 8418             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8419             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8420             "mov    $dst, $tmp\t# vector (1D)" %}
 8421   ins_encode %{
 8422     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8423     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8424               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8425     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8426     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8427     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8428   %}
 8429 
 8430   ins_pipe(pipe_class_default);
 8431 %}
 8432 
 8433 // Note: Long.bitCount(long) returns an int.
 8434 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8435   predicate(UsePopCountInstruction);
 8436   match(Set dst (PopCountL src));
 8437   effect(TEMP tmp);
 8438   ins_cost(INSN_COST * 13);
 8439 
 8440   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8441             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8442             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8443             "mov    $dst, $tmp\t# vector (1D)" %}
 8444   ins_encode %{
 8445     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8446     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8447     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8448     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8449   %}
 8450 
 8451   ins_pipe(pipe_class_default);
 8452 %}
 8453 
 8454 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8455   predicate(UsePopCountInstruction);
 8456   match(Set dst (PopCountL (LoadL mem)));
 8457   effect(TEMP tmp);
 8458   ins_cost(INSN_COST * 13);
 8459 
 8460   format %{ "ldrd   $tmp, $mem\n\t"
 8461             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8462             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8463             "mov    $dst, $tmp\t# vector (1D)" %}
 8464   ins_encode %{
 8465     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8466     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8467               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8468     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8469     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8470     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8471   %}
 8472 
 8473   ins_pipe(pipe_class_default);
 8474 %}
 8475 
 8476 // ============================================================================
 8477 // MemBar Instruction
 8478 
 8479 instruct load_fence() %{
 8480   match(LoadFence);
 8481   ins_cost(VOLATILE_REF_COST);
 8482 
 8483   format %{ "load_fence" %}
 8484 
 8485   ins_encode %{
 8486     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8487   %}
 8488   ins_pipe(pipe_serial);
 8489 %}
 8490 
 8491 instruct unnecessary_membar_acquire() %{
 8492   predicate(unnecessary_acquire(n));
 8493   match(MemBarAcquire);
 8494   ins_cost(0);
 8495 
 8496   format %{ "membar_acquire (elided)" %}
 8497 
 8498   ins_encode %{
 8499     __ block_comment("membar_acquire (elided)");
 8500   %}
 8501 
 8502   ins_pipe(pipe_class_empty);
 8503 %}
 8504 
 8505 instruct membar_acquire() %{
 8506   match(MemBarAcquire);
 8507   ins_cost(VOLATILE_REF_COST);
 8508 
 8509   format %{ "membar_acquire\n\t"
 8510             "dmb ish" %}
 8511 
 8512   ins_encode %{
 8513     __ block_comment("membar_acquire");
 8514     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8515   %}
 8516 
 8517   ins_pipe(pipe_serial);
 8518 %}
 8519 
 8520 
 8521 instruct membar_acquire_lock() %{
 8522   match(MemBarAcquireLock);
 8523   ins_cost(VOLATILE_REF_COST);
 8524 
 8525   format %{ "membar_acquire_lock (elided)" %}
 8526 
 8527   ins_encode %{
 8528     __ block_comment("membar_acquire_lock (elided)");
 8529   %}
 8530 
 8531   ins_pipe(pipe_serial);
 8532 %}
 8533 
 8534 instruct store_fence() %{
 8535   match(StoreFence);
 8536   ins_cost(VOLATILE_REF_COST);
 8537 
 8538   format %{ "store_fence" %}
 8539 
 8540   ins_encode %{
 8541     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8542   %}
 8543   ins_pipe(pipe_serial);
 8544 %}
 8545 
 8546 instruct unnecessary_membar_release() %{
 8547   predicate(unnecessary_release(n));
 8548   match(MemBarRelease);
 8549   ins_cost(0);
 8550 
 8551   format %{ "membar_release (elided)" %}
 8552 
 8553   ins_encode %{
 8554     __ block_comment("membar_release (elided)");
 8555   %}
 8556   ins_pipe(pipe_serial);
 8557 %}
 8558 
 8559 instruct membar_release() %{
 8560   match(MemBarRelease);
 8561   ins_cost(VOLATILE_REF_COST);
 8562 
 8563   format %{ "membar_release\n\t"
 8564             "dmb ish" %}
 8565 
 8566   ins_encode %{
 8567     __ block_comment("membar_release");
 8568     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8569   %}
 8570   ins_pipe(pipe_serial);
 8571 %}
 8572 
 8573 instruct membar_storestore() %{
 8574   match(MemBarStoreStore);
 8575   ins_cost(VOLATILE_REF_COST);
 8576 
 8577   format %{ "MEMBAR-store-store" %}
 8578 
 8579   ins_encode %{
 8580     __ membar(Assembler::StoreStore);
 8581   %}
 8582   ins_pipe(pipe_serial);
 8583 %}
 8584 
 8585 instruct membar_release_lock() %{
 8586   match(MemBarReleaseLock);
 8587   ins_cost(VOLATILE_REF_COST);
 8588 
 8589   format %{ "membar_release_lock (elided)" %}
 8590 
 8591   ins_encode %{
 8592     __ block_comment("membar_release_lock (elided)");
 8593   %}
 8594 
 8595   ins_pipe(pipe_serial);
 8596 %}
 8597 
 8598 instruct unnecessary_membar_volatile() %{
 8599   predicate(unnecessary_volatile(n));
 8600   match(MemBarVolatile);
 8601   ins_cost(0);
 8602 
 8603   format %{ "membar_volatile (elided)" %}
 8604 
 8605   ins_encode %{
 8606     __ block_comment("membar_volatile (elided)");
 8607   %}
 8608 
 8609   ins_pipe(pipe_serial);
 8610 %}
 8611 
 8612 instruct membar_volatile() %{
 8613   match(MemBarVolatile);
 8614   ins_cost(VOLATILE_REF_COST*100);
 8615 
 8616   format %{ "membar_volatile\n\t"
 8617              "dmb ish"%}
 8618 
 8619   ins_encode %{
 8620     __ block_comment("membar_volatile");
 8621     __ membar(Assembler::StoreLoad);
 8622   %}
 8623 
 8624   ins_pipe(pipe_serial);
 8625 %}
 8626 
 8627 // ============================================================================
 8628 // Cast/Convert Instructions
 8629 
 8630 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8631   match(Set dst (CastX2P src));
 8632 
 8633   ins_cost(INSN_COST);
 8634   format %{ "mov $dst, $src\t# long -> ptr" %}
 8635 
 8636   ins_encode %{
 8637     if ($dst$$reg != $src$$reg) {
 8638       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8639     }
 8640   %}
 8641 
 8642   ins_pipe(ialu_reg);
 8643 %}
 8644 
 8645 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8646   match(Set dst (CastP2X src));
 8647 
 8648   ins_cost(INSN_COST);
 8649   format %{ "mov $dst, $src\t# ptr -> long" %}
 8650 
 8651   ins_encode %{
 8652     if ($dst$$reg != $src$$reg) {
 8653       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8654     }
 8655   %}
 8656 
 8657   ins_pipe(ialu_reg);
 8658 %}
 8659 
 8660 // Convert oop into int for vectors alignment masking
 8661 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8662   match(Set dst (ConvL2I (CastP2X src)));
 8663 
 8664   ins_cost(INSN_COST);
 8665   format %{ "movw $dst, $src\t# ptr -> int" %}
 8666   ins_encode %{
 8667     __ movw($dst$$Register, $src$$Register);
 8668   %}
 8669 
 8670   ins_pipe(ialu_reg);
 8671 %}
 8672 
 8673 // Convert compressed oop into int for vectors alignment masking
 8674 // in case of 32bit oops (heap < 4Gb).
 8675 instruct convN2I(iRegINoSp dst, iRegN src)
 8676 %{
 8677   predicate(CompressedOops::shift() == 0);
 8678   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8679 
 8680   ins_cost(INSN_COST);
 8681   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8682   ins_encode %{
 8683     __ movw($dst$$Register, $src$$Register);
 8684   %}
 8685 
 8686   ins_pipe(ialu_reg);
 8687 %}
 8688 
 8689 
 8690 // Convert oop pointer into compressed form
 8691 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8692   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8693   match(Set dst (EncodeP src));
 8694   effect(KILL cr);
 8695   ins_cost(INSN_COST * 3);
 8696   format %{ "encode_heap_oop $dst, $src" %}
 8697   ins_encode %{
 8698     Register s = $src$$Register;
 8699     Register d = $dst$$Register;
 8700     __ encode_heap_oop(d, s);
 8701   %}
 8702   ins_pipe(ialu_reg);
 8703 %}
 8704 
 8705 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8706   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8707   match(Set dst (EncodeP src));
 8708   ins_cost(INSN_COST * 3);
 8709   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8710   ins_encode %{
 8711     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8712   %}
 8713   ins_pipe(ialu_reg);
 8714 %}
 8715 
 8716 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8717   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8718             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8719   match(Set dst (DecodeN src));
 8720   ins_cost(INSN_COST * 3);
 8721   format %{ "decode_heap_oop $dst, $src" %}
 8722   ins_encode %{
 8723     Register s = $src$$Register;
 8724     Register d = $dst$$Register;
 8725     __ decode_heap_oop(d, s);
 8726   %}
 8727   ins_pipe(ialu_reg);
 8728 %}
 8729 
 8730 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8731   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8732             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8733   match(Set dst (DecodeN src));
 8734   ins_cost(INSN_COST * 3);
 8735   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8736   ins_encode %{
 8737     Register s = $src$$Register;
 8738     Register d = $dst$$Register;
 8739     __ decode_heap_oop_not_null(d, s);
 8740   %}
 8741   ins_pipe(ialu_reg);
 8742 %}
 8743 
 8744 // n.b. AArch64 implementations of encode_klass_not_null and
 8745 // decode_klass_not_null do not modify the flags register so, unlike
 8746 // Intel, we don't kill CR as a side effect here
 8747 
 8748 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8749   match(Set dst (EncodePKlass src));
 8750 
 8751   ins_cost(INSN_COST * 3);
 8752   format %{ "encode_klass_not_null $dst,$src" %}
 8753 
 8754   ins_encode %{
 8755     Register src_reg = as_Register($src$$reg);
 8756     Register dst_reg = as_Register($dst$$reg);
 8757     __ encode_klass_not_null(dst_reg, src_reg);
 8758   %}
 8759 
 8760    ins_pipe(ialu_reg);
 8761 %}
 8762 
 8763 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8764   match(Set dst (DecodeNKlass src));
 8765 
 8766   ins_cost(INSN_COST * 3);
 8767   format %{ "decode_klass_not_null $dst,$src" %}
 8768 
 8769   ins_encode %{
 8770     Register src_reg = as_Register($src$$reg);
 8771     Register dst_reg = as_Register($dst$$reg);
 8772     if (dst_reg != src_reg) {
 8773       __ decode_klass_not_null(dst_reg, src_reg);
 8774     } else {
 8775       __ decode_klass_not_null(dst_reg);
 8776     }
 8777   %}
 8778 
 8779    ins_pipe(ialu_reg);
 8780 %}
 8781 
 8782 instruct checkCastPP(iRegPNoSp dst)
 8783 %{
 8784   match(Set dst (CheckCastPP dst));
 8785 
 8786   size(0);
 8787   format %{ "# checkcastPP of $dst" %}
 8788   ins_encode(/* empty encoding */);
 8789   ins_pipe(pipe_class_empty);
 8790 %}
 8791 
 8792 instruct castPP(iRegPNoSp dst)
 8793 %{
 8794   match(Set dst (CastPP dst));
 8795 
 8796   size(0);
 8797   format %{ "# castPP of $dst" %}
 8798   ins_encode(/* empty encoding */);
 8799   ins_pipe(pipe_class_empty);
 8800 %}
 8801 
 8802 instruct castII(iRegI dst)
 8803 %{
 8804   match(Set dst (CastII dst));
 8805 
 8806   size(0);
 8807   format %{ "# castII of $dst" %}
 8808   ins_encode(/* empty encoding */);
 8809   ins_cost(0);
 8810   ins_pipe(pipe_class_empty);
 8811 %}
 8812 
 8813 instruct castLL(iRegL dst)
 8814 %{
 8815   match(Set dst (CastLL dst));
 8816 
 8817   size(0);
 8818   format %{ "# castLL of $dst" %}
 8819   ins_encode(/* empty encoding */);
 8820   ins_cost(0);
 8821   ins_pipe(pipe_class_empty);
 8822 %}
 8823 
 8824 instruct castFF(vRegF dst)
 8825 %{
 8826   match(Set dst (CastFF dst));
 8827 
 8828   size(0);
 8829   format %{ "# castFF of $dst" %}
 8830   ins_encode(/* empty encoding */);
 8831   ins_cost(0);
 8832   ins_pipe(pipe_class_empty);
 8833 %}
 8834 
 8835 instruct castDD(vRegD dst)
 8836 %{
 8837   match(Set dst (CastDD dst));
 8838 
 8839   size(0);
 8840   format %{ "# castDD of $dst" %}
 8841   ins_encode(/* empty encoding */);
 8842   ins_cost(0);
 8843   ins_pipe(pipe_class_empty);
 8844 %}
 8845 
 8846 instruct castVVD(vecD dst)
 8847 %{
 8848   match(Set dst (CastVV dst));
 8849 
 8850   size(0);
 8851   format %{ "# castVV of $dst" %}
 8852   ins_encode(/* empty encoding */);
 8853   ins_cost(0);
 8854   ins_pipe(pipe_class_empty);
 8855 %}
 8856 
 8857 instruct castVVX(vecX dst)
 8858 %{
 8859   match(Set dst (CastVV dst));
 8860 
 8861   size(0);
 8862   format %{ "# castVV of $dst" %}
 8863   ins_encode(/* empty encoding */);
 8864   ins_cost(0);
 8865   ins_pipe(pipe_class_empty);
 8866 %}
 8867 
 8868 instruct castVV(vReg dst)
 8869 %{
 8870   match(Set dst (CastVV dst));
 8871 
 8872   size(0);
 8873   format %{ "# castVV of $dst" %}
 8874   ins_encode(/* empty encoding */);
 8875   ins_cost(0);
 8876   ins_pipe(pipe_class_empty);
 8877 %}
 8878 
 8879 // ============================================================================
 8880 // Atomic operation instructions
 8881 //
 8882 // Intel and SPARC both implement Ideal Node LoadPLocked and
 8883 // Store{PIL}Conditional instructions using a normal load for the
 8884 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 8885 //
 8886 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 8887 // pair to lock object allocations from Eden space when not using
 8888 // TLABs.
 8889 //
 8890 // There does not appear to be a Load{IL}Locked Ideal Node and the
 8891 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 8892 // and to use StoreIConditional only for 32-bit and StoreLConditional
 8893 // only for 64-bit.
 8894 //
 8895 // We implement LoadPLocked and StorePLocked instructions using,
 8896 // respectively the AArch64 hw load-exclusive and store-conditional
 8897 // instructions. Whereas we must implement each of
 8898 // Store{IL}Conditional using a CAS which employs a pair of
 8899 // instructions comprising a load-exclusive followed by a
 8900 // store-conditional.
 8901 
 8902 
 8903 // Locked-load (linked load) of the current heap-top
 8904 // used when updating the eden heap top
 8905 // implemented using ldaxr on AArch64
 8906 
 8907 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 8908 %{
 8909   match(Set dst (LoadPLocked mem));
 8910 
 8911   ins_cost(VOLATILE_REF_COST);
 8912 
 8913   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 8914 
 8915   ins_encode(aarch64_enc_ldaxr(dst, mem));
 8916 
 8917   ins_pipe(pipe_serial);
 8918 %}
 8919 
 8920 // Conditional-store of the updated heap-top.
 8921 // Used during allocation of the shared heap.
 8922 // Sets flag (EQ) on success.
 8923 // implemented using stlxr on AArch64.
 8924 
 8925 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 8926 %{
 8927   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 8928 
 8929   ins_cost(VOLATILE_REF_COST);
 8930 
 8931  // TODO
 8932  // do we need to do a store-conditional release or can we just use a
 8933  // plain store-conditional?
 8934 
 8935   format %{
 8936     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 8937     "cmpw rscratch1, zr\t# EQ on successful write"
 8938   %}
 8939 
 8940   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 8941 
 8942   ins_pipe(pipe_serial);
 8943 %}
 8944 
 8945 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 8946 %{
 8947   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 8948 
 8949   ins_cost(VOLATILE_REF_COST);
 8950 
 8951   format %{
 8952     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 8953     "cmpw rscratch1, zr\t# EQ on successful write"
 8954   %}
 8955 
 8956   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 8957 
 8958   ins_pipe(pipe_slow);
 8959 %}
 8960 
 8961 // storeIConditional also has acquire semantics, for no better reason
 8962 // than matching storeLConditional.  At the time of writing this
 8963 // comment storeIConditional was not used anywhere by AArch64.
 8964 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 8965 %{
 8966   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 8967 
 8968   ins_cost(VOLATILE_REF_COST);
 8969 
 8970   format %{
 8971     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 8972     "cmpw rscratch1, zr\t# EQ on successful write"
 8973   %}
 8974 
 8975   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 8976 
 8977   ins_pipe(pipe_slow);
 8978 %}
 8979 
 8980 // standard CompareAndSwapX when we are using barriers
 8981 // these have higher priority than the rules selected by a predicate
 8982 
 8983 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8984 // can't match them
 8985 
 8986 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8987 
 8988   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8989   ins_cost(2 * VOLATILE_REF_COST);
 8990 
 8991   effect(KILL cr);
 8992 
 8993   format %{
 8994     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8995     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8996   %}
 8997 
 8998   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8999             aarch64_enc_cset_eq(res));
 9000 
 9001   ins_pipe(pipe_slow);
 9002 %}
 9003 
 9004 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9005 
 9006   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9007   ins_cost(2 * VOLATILE_REF_COST);
 9008 
 9009   effect(KILL cr);
 9010 
 9011   format %{
 9012     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9013     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9014   %}
 9015 
 9016   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9017             aarch64_enc_cset_eq(res));
 9018 
 9019   ins_pipe(pipe_slow);
 9020 %}
 9021 
 9022 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9023 
 9024   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9025   ins_cost(2 * VOLATILE_REF_COST);
 9026 
 9027   effect(KILL cr);
 9028 
 9029  format %{
 9030     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9031     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9032  %}
 9033 
 9034  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9035             aarch64_enc_cset_eq(res));
 9036 
 9037   ins_pipe(pipe_slow);
 9038 %}
 9039 
 9040 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9041 
 9042   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9043   ins_cost(2 * VOLATILE_REF_COST);
 9044 
 9045   effect(KILL cr);
 9046 
 9047  format %{
 9048     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9049     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9050  %}
 9051 
 9052  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9053             aarch64_enc_cset_eq(res));
 9054 
 9055   ins_pipe(pipe_slow);
 9056 %}
 9057 
 9058 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9059 
 9060   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9061   predicate(n->as_LoadStore()->barrier_data() == 0);
 9062   ins_cost(2 * VOLATILE_REF_COST);
 9063 
 9064   effect(KILL cr);
 9065 
 9066  format %{
 9067     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9068     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9069  %}
 9070 
 9071  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9072             aarch64_enc_cset_eq(res));
 9073 
 9074   ins_pipe(pipe_slow);
 9075 %}
 9076 
 9077 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9078 
 9079   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9080   ins_cost(2 * VOLATILE_REF_COST);
 9081 
 9082   effect(KILL cr);
 9083 
 9084  format %{
 9085     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9086     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9087  %}
 9088 
 9089  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9090             aarch64_enc_cset_eq(res));
 9091 
 9092   ins_pipe(pipe_slow);
 9093 %}
 9094 
 9095 // alternative CompareAndSwapX when we are eliding barriers
 9096 
 9097 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9098 
 9099   predicate(needs_acquiring_load_exclusive(n));
 9100   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9101   ins_cost(VOLATILE_REF_COST);
 9102 
 9103   effect(KILL cr);
 9104 
 9105   format %{
 9106     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9107     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9108   %}
 9109 
 9110   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9111             aarch64_enc_cset_eq(res));
 9112 
 9113   ins_pipe(pipe_slow);
 9114 %}
 9115 
 9116 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9117 
 9118   predicate(needs_acquiring_load_exclusive(n));
 9119   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9120   ins_cost(VOLATILE_REF_COST);
 9121 
 9122   effect(KILL cr);
 9123 
 9124   format %{
 9125     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9126     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9127   %}
 9128 
 9129   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9130             aarch64_enc_cset_eq(res));
 9131 
 9132   ins_pipe(pipe_slow);
 9133 %}
 9134 
 9135 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9136 
 9137   predicate(needs_acquiring_load_exclusive(n));
 9138   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9139   ins_cost(VOLATILE_REF_COST);
 9140 
 9141   effect(KILL cr);
 9142 
 9143  format %{
 9144     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9145     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9146  %}
 9147 
 9148  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9149             aarch64_enc_cset_eq(res));
 9150 
 9151   ins_pipe(pipe_slow);
 9152 %}
 9153 
 9154 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9155 
 9156   predicate(needs_acquiring_load_exclusive(n));
 9157   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9158   ins_cost(VOLATILE_REF_COST);
 9159 
 9160   effect(KILL cr);
 9161 
 9162  format %{
 9163     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9164     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9165  %}
 9166 
 9167  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9168             aarch64_enc_cset_eq(res));
 9169 
 9170   ins_pipe(pipe_slow);
 9171 %}
 9172 
 9173 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9174 
 9175   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9176   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9177   ins_cost(VOLATILE_REF_COST);
 9178 
 9179   effect(KILL cr);
 9180 
 9181  format %{
 9182     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9183     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9184  %}
 9185 
 9186  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9187             aarch64_enc_cset_eq(res));
 9188 
 9189   ins_pipe(pipe_slow);
 9190 %}
 9191 
 9192 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9193 
 9194   predicate(needs_acquiring_load_exclusive(n));
 9195   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9196   ins_cost(VOLATILE_REF_COST);
 9197 
 9198   effect(KILL cr);
 9199 
 9200  format %{
 9201     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9202     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9203  %}
 9204 
 9205  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9206             aarch64_enc_cset_eq(res));
 9207 
 9208   ins_pipe(pipe_slow);
 9209 %}
 9210 
 9211 
 9212 // ---------------------------------------------------------------------
 9213 
 9214 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9215 
 9216 // Sundry CAS operations.  Note that release is always true,
 9217 // regardless of the memory ordering of the CAS.  This is because we
 9218 // need the volatile case to be sequentially consistent but there is
 9219 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9220 // can't check the type of memory ordering here, so we always emit a
 9221 // STLXR.
 9222 
 9223 // This section is generated from aarch64_ad_cas.m4
 9224 
 9225 
 9226 
 9227 // This pattern is generated automatically from cas.m4.
 9228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9229 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9230 
 9231   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9232   ins_cost(2 * VOLATILE_REF_COST);
 9233   effect(TEMP_DEF res, KILL cr);
 9234   format %{
 9235     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9236   %}
 9237   ins_encode %{
 9238     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9239                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9240                /*weak*/ false, $res$$Register);
 9241     __ sxtbw($res$$Register, $res$$Register);
 9242   %}
 9243   ins_pipe(pipe_slow);
 9244 %}
 9245 
 9246 // This pattern is generated automatically from cas.m4.
 9247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9248 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9249 
 9250   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9251   ins_cost(2 * VOLATILE_REF_COST);
 9252   effect(TEMP_DEF res, KILL cr);
 9253   format %{
 9254     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9255   %}
 9256   ins_encode %{
 9257     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9258                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9259                /*weak*/ false, $res$$Register);
 9260     __ sxthw($res$$Register, $res$$Register);
 9261   %}
 9262   ins_pipe(pipe_slow);
 9263 %}
 9264 
 9265 // This pattern is generated automatically from cas.m4.
 9266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9267 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9268 
 9269   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9270   ins_cost(2 * VOLATILE_REF_COST);
 9271   effect(TEMP_DEF res, KILL cr);
 9272   format %{
 9273     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9274   %}
 9275   ins_encode %{
 9276     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9277                Assembler::word, /*acquire*/ false, /*release*/ true,
 9278                /*weak*/ false, $res$$Register);
 9279   %}
 9280   ins_pipe(pipe_slow);
 9281 %}
 9282 
 9283 // This pattern is generated automatically from cas.m4.
 9284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9285 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9286 
 9287   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9288   ins_cost(2 * VOLATILE_REF_COST);
 9289   effect(TEMP_DEF res, KILL cr);
 9290   format %{
 9291     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9292   %}
 9293   ins_encode %{
 9294     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9295                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9296                /*weak*/ false, $res$$Register);
 9297   %}
 9298   ins_pipe(pipe_slow);
 9299 %}
 9300 
 9301 // This pattern is generated automatically from cas.m4.
 9302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9303 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9304 
 9305   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9306   ins_cost(2 * VOLATILE_REF_COST);
 9307   effect(TEMP_DEF res, KILL cr);
 9308   format %{
 9309     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9310   %}
 9311   ins_encode %{
 9312     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9313                Assembler::word, /*acquire*/ false, /*release*/ true,
 9314                /*weak*/ false, $res$$Register);
 9315   %}
 9316   ins_pipe(pipe_slow);
 9317 %}
 9318 
 9319 // This pattern is generated automatically from cas.m4.
 9320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9321 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9322   predicate(n->as_LoadStore()->barrier_data() == 0);
 9323   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9324   ins_cost(2 * VOLATILE_REF_COST);
 9325   effect(TEMP_DEF res, KILL cr);
 9326   format %{
 9327     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9328   %}
 9329   ins_encode %{
 9330     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9331                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9332                /*weak*/ false, $res$$Register);
 9333   %}
 9334   ins_pipe(pipe_slow);
 9335 %}
 9336 
 9337 // This pattern is generated automatically from cas.m4.
 9338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9339 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9340   predicate(needs_acquiring_load_exclusive(n));
 9341   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9342   ins_cost(VOLATILE_REF_COST);
 9343   effect(TEMP_DEF res, KILL cr);
 9344   format %{
 9345     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9346   %}
 9347   ins_encode %{
 9348     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9349                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9350                /*weak*/ false, $res$$Register);
 9351     __ sxtbw($res$$Register, $res$$Register);
 9352   %}
 9353   ins_pipe(pipe_slow);
 9354 %}
 9355 
 9356 // This pattern is generated automatically from cas.m4.
 9357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9358 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9359   predicate(needs_acquiring_load_exclusive(n));
 9360   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9361   ins_cost(VOLATILE_REF_COST);
 9362   effect(TEMP_DEF res, KILL cr);
 9363   format %{
 9364     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9365   %}
 9366   ins_encode %{
 9367     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9368                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9369                /*weak*/ false, $res$$Register);
 9370     __ sxthw($res$$Register, $res$$Register);
 9371   %}
 9372   ins_pipe(pipe_slow);
 9373 %}
 9374 
 9375 // This pattern is generated automatically from cas.m4.
 9376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9377 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9378   predicate(needs_acquiring_load_exclusive(n));
 9379   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9380   ins_cost(VOLATILE_REF_COST);
 9381   effect(TEMP_DEF res, KILL cr);
 9382   format %{
 9383     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9384   %}
 9385   ins_encode %{
 9386     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9387                Assembler::word, /*acquire*/ true, /*release*/ true,
 9388                /*weak*/ false, $res$$Register);
 9389   %}
 9390   ins_pipe(pipe_slow);
 9391 %}
 9392 
 9393 // This pattern is generated automatically from cas.m4.
 9394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9395 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9396   predicate(needs_acquiring_load_exclusive(n));
 9397   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9398   ins_cost(VOLATILE_REF_COST);
 9399   effect(TEMP_DEF res, KILL cr);
 9400   format %{
 9401     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9402   %}
 9403   ins_encode %{
 9404     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9405                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9406                /*weak*/ false, $res$$Register);
 9407   %}
 9408   ins_pipe(pipe_slow);
 9409 %}
 9410 
 9411 // This pattern is generated automatically from cas.m4.
 9412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9413 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9414   predicate(needs_acquiring_load_exclusive(n));
 9415   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9416   ins_cost(VOLATILE_REF_COST);
 9417   effect(TEMP_DEF res, KILL cr);
 9418   format %{
 9419     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9420   %}
 9421   ins_encode %{
 9422     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9423                Assembler::word, /*acquire*/ true, /*release*/ true,
 9424                /*weak*/ false, $res$$Register);
 9425   %}
 9426   ins_pipe(pipe_slow);
 9427 %}
 9428 
 9429 // This pattern is generated automatically from cas.m4.
 9430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9431 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9432   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9433   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9434   ins_cost(VOLATILE_REF_COST);
 9435   effect(TEMP_DEF res, KILL cr);
 9436   format %{
 9437     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9438   %}
 9439   ins_encode %{
 9440     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9441                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9442                /*weak*/ false, $res$$Register);
 9443   %}
 9444   ins_pipe(pipe_slow);
 9445 %}
 9446 
 9447 // This pattern is generated automatically from cas.m4.
 9448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9449 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9450 
 9451   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9452   ins_cost(2 * VOLATILE_REF_COST);
 9453   effect(KILL cr);
 9454   format %{
 9455     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9456     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9457   %}
 9458   ins_encode %{
 9459     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9460                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9461                /*weak*/ true, noreg);
 9462     __ csetw($res$$Register, Assembler::EQ);
 9463   %}
 9464   ins_pipe(pipe_slow);
 9465 %}
 9466 
 9467 // This pattern is generated automatically from cas.m4.
 9468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9469 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9470 
 9471   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9472   ins_cost(2 * VOLATILE_REF_COST);
 9473   effect(KILL cr);
 9474   format %{
 9475     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9476     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9477   %}
 9478   ins_encode %{
 9479     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9480                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9481                /*weak*/ true, noreg);
 9482     __ csetw($res$$Register, Assembler::EQ);
 9483   %}
 9484   ins_pipe(pipe_slow);
 9485 %}
 9486 
 9487 // This pattern is generated automatically from cas.m4.
 9488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9489 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9490 
 9491   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9492   ins_cost(2 * VOLATILE_REF_COST);
 9493   effect(KILL cr);
 9494   format %{
 9495     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9496     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9497   %}
 9498   ins_encode %{
 9499     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9500                Assembler::word, /*acquire*/ false, /*release*/ true,
 9501                /*weak*/ true, noreg);
 9502     __ csetw($res$$Register, Assembler::EQ);
 9503   %}
 9504   ins_pipe(pipe_slow);
 9505 %}
 9506 
 9507 // This pattern is generated automatically from cas.m4.
 9508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9509 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9510 
 9511   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9512   ins_cost(2 * VOLATILE_REF_COST);
 9513   effect(KILL cr);
 9514   format %{
 9515     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9516     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9517   %}
 9518   ins_encode %{
 9519     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9520                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9521                /*weak*/ true, noreg);
 9522     __ csetw($res$$Register, Assembler::EQ);
 9523   %}
 9524   ins_pipe(pipe_slow);
 9525 %}
 9526 
 9527 // This pattern is generated automatically from cas.m4.
 9528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9529 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9530 
 9531   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9532   ins_cost(2 * VOLATILE_REF_COST);
 9533   effect(KILL cr);
 9534   format %{
 9535     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9536     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9537   %}
 9538   ins_encode %{
 9539     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9540                Assembler::word, /*acquire*/ false, /*release*/ true,
 9541                /*weak*/ true, noreg);
 9542     __ csetw($res$$Register, Assembler::EQ);
 9543   %}
 9544   ins_pipe(pipe_slow);
 9545 %}
 9546 
 9547 // This pattern is generated automatically from cas.m4.
 9548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9549 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9550   predicate(n->as_LoadStore()->barrier_data() == 0);
 9551   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9552   ins_cost(2 * VOLATILE_REF_COST);
 9553   effect(KILL cr);
 9554   format %{
 9555     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9556     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9557   %}
 9558   ins_encode %{
 9559     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9560                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9561                /*weak*/ true, noreg);
 9562     __ csetw($res$$Register, Assembler::EQ);
 9563   %}
 9564   ins_pipe(pipe_slow);
 9565 %}
 9566 
 9567 // This pattern is generated automatically from cas.m4.
 9568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9569 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9570   predicate(needs_acquiring_load_exclusive(n));
 9571   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9572   ins_cost(VOLATILE_REF_COST);
 9573   effect(KILL cr);
 9574   format %{
 9575     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9576     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9577   %}
 9578   ins_encode %{
 9579     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9580                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9581                /*weak*/ true, noreg);
 9582     __ csetw($res$$Register, Assembler::EQ);
 9583   %}
 9584   ins_pipe(pipe_slow);
 9585 %}
 9586 
 9587 // This pattern is generated automatically from cas.m4.
 9588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9589 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9590   predicate(needs_acquiring_load_exclusive(n));
 9591   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9592   ins_cost(VOLATILE_REF_COST);
 9593   effect(KILL cr);
 9594   format %{
 9595     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9596     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9597   %}
 9598   ins_encode %{
 9599     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9600                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9601                /*weak*/ true, noreg);
 9602     __ csetw($res$$Register, Assembler::EQ);
 9603   %}
 9604   ins_pipe(pipe_slow);
 9605 %}
 9606 
 9607 // This pattern is generated automatically from cas.m4.
 9608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9609 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9610   predicate(needs_acquiring_load_exclusive(n));
 9611   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9612   ins_cost(VOLATILE_REF_COST);
 9613   effect(KILL cr);
 9614   format %{
 9615     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9616     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9617   %}
 9618   ins_encode %{
 9619     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9620                Assembler::word, /*acquire*/ true, /*release*/ true,
 9621                /*weak*/ true, noreg);
 9622     __ csetw($res$$Register, Assembler::EQ);
 9623   %}
 9624   ins_pipe(pipe_slow);
 9625 %}
 9626 
 9627 // This pattern is generated automatically from cas.m4.
 9628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9629 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9630   predicate(needs_acquiring_load_exclusive(n));
 9631   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9632   ins_cost(VOLATILE_REF_COST);
 9633   effect(KILL cr);
 9634   format %{
 9635     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9636     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9637   %}
 9638   ins_encode %{
 9639     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9640                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9641                /*weak*/ true, noreg);
 9642     __ csetw($res$$Register, Assembler::EQ);
 9643   %}
 9644   ins_pipe(pipe_slow);
 9645 %}
 9646 
 9647 // This pattern is generated automatically from cas.m4.
 9648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9649 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9650   predicate(needs_acquiring_load_exclusive(n));
 9651   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9652   ins_cost(VOLATILE_REF_COST);
 9653   effect(KILL cr);
 9654   format %{
 9655     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9656     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9657   %}
 9658   ins_encode %{
 9659     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9660                Assembler::word, /*acquire*/ true, /*release*/ true,
 9661                /*weak*/ true, noreg);
 9662     __ csetw($res$$Register, Assembler::EQ);
 9663   %}
 9664   ins_pipe(pipe_slow);
 9665 %}
 9666 
 9667 // This pattern is generated automatically from cas.m4.
 9668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9669 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9670   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9671   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9672   ins_cost(VOLATILE_REF_COST);
 9673   effect(KILL cr);
 9674   format %{
 9675     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9676     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9677   %}
 9678   ins_encode %{
 9679     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9680                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9681                /*weak*/ true, noreg);
 9682     __ csetw($res$$Register, Assembler::EQ);
 9683   %}
 9684   ins_pipe(pipe_slow);
 9685 %}
 9686 
 9687 // END This section of the file is automatically generated. Do not edit --------------
 9688 // ---------------------------------------------------------------------
 9689 
 9690 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9691   match(Set prev (GetAndSetI mem newv));
 9692   ins_cost(2 * VOLATILE_REF_COST);
 9693   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9694   ins_encode %{
 9695     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9696   %}
 9697   ins_pipe(pipe_serial);
 9698 %}
 9699 
 9700 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9701   match(Set prev (GetAndSetL mem newv));
 9702   ins_cost(2 * VOLATILE_REF_COST);
 9703   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9704   ins_encode %{
 9705     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9706   %}
 9707   ins_pipe(pipe_serial);
 9708 %}
 9709 
 9710 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9711   match(Set prev (GetAndSetN mem newv));
 9712   ins_cost(2 * VOLATILE_REF_COST);
 9713   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9714   ins_encode %{
 9715     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9716   %}
 9717   ins_pipe(pipe_serial);
 9718 %}
 9719 
 9720 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9721   predicate(n->as_LoadStore()->barrier_data() == 0);
 9722   match(Set prev (GetAndSetP mem newv));
 9723   ins_cost(2 * VOLATILE_REF_COST);
 9724   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9725   ins_encode %{
 9726     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9727   %}
 9728   ins_pipe(pipe_serial);
 9729 %}
 9730 
 9731 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9732   predicate(needs_acquiring_load_exclusive(n));
 9733   match(Set prev (GetAndSetI mem newv));
 9734   ins_cost(VOLATILE_REF_COST);
 9735   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9736   ins_encode %{
 9737     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9738   %}
 9739   ins_pipe(pipe_serial);
 9740 %}
 9741 
 9742 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9743   predicate(needs_acquiring_load_exclusive(n));
 9744   match(Set prev (GetAndSetL mem newv));
 9745   ins_cost(VOLATILE_REF_COST);
 9746   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9747   ins_encode %{
 9748     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9749   %}
 9750   ins_pipe(pipe_serial);
 9751 %}
 9752 
 9753 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9754   predicate(needs_acquiring_load_exclusive(n));
 9755   match(Set prev (GetAndSetN mem newv));
 9756   ins_cost(VOLATILE_REF_COST);
 9757   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9758   ins_encode %{
 9759     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9760   %}
 9761   ins_pipe(pipe_serial);
 9762 %}
 9763 
 9764 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9765   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9766   match(Set prev (GetAndSetP mem newv));
 9767   ins_cost(VOLATILE_REF_COST);
 9768   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9769   ins_encode %{
 9770     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9771   %}
 9772   ins_pipe(pipe_serial);
 9773 %}
 9774 
 9775 
 9776 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9777   match(Set newval (GetAndAddL mem incr));
 9778   ins_cost(2 * VOLATILE_REF_COST + 1);
 9779   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9780   ins_encode %{
 9781     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9782   %}
 9783   ins_pipe(pipe_serial);
 9784 %}
 9785 
 9786 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9787   predicate(n->as_LoadStore()->result_not_used());
 9788   match(Set dummy (GetAndAddL mem incr));
 9789   ins_cost(2 * VOLATILE_REF_COST);
 9790   format %{ "get_and_addL [$mem], $incr" %}
 9791   ins_encode %{
 9792     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9793   %}
 9794   ins_pipe(pipe_serial);
 9795 %}
 9796 
 9797 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9798   match(Set newval (GetAndAddL mem incr));
 9799   ins_cost(2 * VOLATILE_REF_COST + 1);
 9800   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9801   ins_encode %{
 9802     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9803   %}
 9804   ins_pipe(pipe_serial);
 9805 %}
 9806 
 9807 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9808   predicate(n->as_LoadStore()->result_not_used());
 9809   match(Set dummy (GetAndAddL mem incr));
 9810   ins_cost(2 * VOLATILE_REF_COST);
 9811   format %{ "get_and_addL [$mem], $incr" %}
 9812   ins_encode %{
 9813     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9814   %}
 9815   ins_pipe(pipe_serial);
 9816 %}
 9817 
 9818 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9819   match(Set newval (GetAndAddI mem incr));
 9820   ins_cost(2 * VOLATILE_REF_COST + 1);
 9821   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9822   ins_encode %{
 9823     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9824   %}
 9825   ins_pipe(pipe_serial);
 9826 %}
 9827 
 9828 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9829   predicate(n->as_LoadStore()->result_not_used());
 9830   match(Set dummy (GetAndAddI mem incr));
 9831   ins_cost(2 * VOLATILE_REF_COST);
 9832   format %{ "get_and_addI [$mem], $incr" %}
 9833   ins_encode %{
 9834     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9835   %}
 9836   ins_pipe(pipe_serial);
 9837 %}
 9838 
 9839 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9840   match(Set newval (GetAndAddI mem incr));
 9841   ins_cost(2 * VOLATILE_REF_COST + 1);
 9842   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9843   ins_encode %{
 9844     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9845   %}
 9846   ins_pipe(pipe_serial);
 9847 %}
 9848 
 9849 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9850   predicate(n->as_LoadStore()->result_not_used());
 9851   match(Set dummy (GetAndAddI mem incr));
 9852   ins_cost(2 * VOLATILE_REF_COST);
 9853   format %{ "get_and_addI [$mem], $incr" %}
 9854   ins_encode %{
 9855     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9856   %}
 9857   ins_pipe(pipe_serial);
 9858 %}
 9859 
 9860 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9861   predicate(needs_acquiring_load_exclusive(n));
 9862   match(Set newval (GetAndAddL mem incr));
 9863   ins_cost(VOLATILE_REF_COST + 1);
 9864   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9865   ins_encode %{
 9866     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9867   %}
 9868   ins_pipe(pipe_serial);
 9869 %}
 9870 
 9871 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9872   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9873   match(Set dummy (GetAndAddL mem incr));
 9874   ins_cost(VOLATILE_REF_COST);
 9875   format %{ "get_and_addL_acq [$mem], $incr" %}
 9876   ins_encode %{
 9877     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9878   %}
 9879   ins_pipe(pipe_serial);
 9880 %}
 9881 
 9882 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9883   predicate(needs_acquiring_load_exclusive(n));
 9884   match(Set newval (GetAndAddL mem incr));
 9885   ins_cost(VOLATILE_REF_COST + 1);
 9886   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9887   ins_encode %{
 9888     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9889   %}
 9890   ins_pipe(pipe_serial);
 9891 %}
 9892 
 9893 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9894   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9895   match(Set dummy (GetAndAddL mem incr));
 9896   ins_cost(VOLATILE_REF_COST);
 9897   format %{ "get_and_addL_acq [$mem], $incr" %}
 9898   ins_encode %{
 9899     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9900   %}
 9901   ins_pipe(pipe_serial);
 9902 %}
 9903 
 9904 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9905   predicate(needs_acquiring_load_exclusive(n));
 9906   match(Set newval (GetAndAddI mem incr));
 9907   ins_cost(VOLATILE_REF_COST + 1);
 9908   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9909   ins_encode %{
 9910     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9911   %}
 9912   ins_pipe(pipe_serial);
 9913 %}
 9914 
 9915 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9916   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9917   match(Set dummy (GetAndAddI mem incr));
 9918   ins_cost(VOLATILE_REF_COST);
 9919   format %{ "get_and_addI_acq [$mem], $incr" %}
 9920   ins_encode %{
 9921     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9922   %}
 9923   ins_pipe(pipe_serial);
 9924 %}
 9925 
 9926 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9927   predicate(needs_acquiring_load_exclusive(n));
 9928   match(Set newval (GetAndAddI mem incr));
 9929   ins_cost(VOLATILE_REF_COST + 1);
 9930   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9931   ins_encode %{
 9932     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9933   %}
 9934   ins_pipe(pipe_serial);
 9935 %}
 9936 
 9937 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9938   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9939   match(Set dummy (GetAndAddI mem incr));
 9940   ins_cost(VOLATILE_REF_COST);
 9941   format %{ "get_and_addI_acq [$mem], $incr" %}
 9942   ins_encode %{
 9943     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9944   %}
 9945   ins_pipe(pipe_serial);
 9946 %}
 9947 
 9948 // Manifest a CmpL result in an integer register.
 9949 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9950 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9951 %{
 9952   match(Set dst (CmpL3 src1 src2));
 9953   effect(KILL flags);
 9954 
 9955   ins_cost(INSN_COST * 6);
 9956   format %{
 9957       "cmp $src1, $src2"
 9958       "csetw $dst, ne"
 9959       "cnegw $dst, lt"
 9960   %}
 9961   // format %{ "CmpL3 $dst, $src1, $src2" %}
 9962   ins_encode %{
 9963     __ cmp($src1$$Register, $src2$$Register);
 9964     __ csetw($dst$$Register, Assembler::NE);
 9965     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9966   %}
 9967 
 9968   ins_pipe(pipe_class_default);
 9969 %}
 9970 
 9971 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9972 %{
 9973   match(Set dst (CmpL3 src1 src2));
 9974   effect(KILL flags);
 9975 
 9976   ins_cost(INSN_COST * 6);
 9977   format %{
 9978       "cmp $src1, $src2"
 9979       "csetw $dst, ne"
 9980       "cnegw $dst, lt"
 9981   %}
 9982   ins_encode %{
 9983     int32_t con = (int32_t)$src2$$constant;
 9984      if (con < 0) {
 9985       __ adds(zr, $src1$$Register, -con);
 9986     } else {
 9987       __ subs(zr, $src1$$Register, con);
 9988     }
 9989     __ csetw($dst$$Register, Assembler::NE);
 9990     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9991   %}
 9992 
 9993   ins_pipe(pipe_class_default);
 9994 %}
 9995 
 9996 // ============================================================================
 9997 // Conditional Move Instructions
 9998 
 9999 // n.b. we have identical rules for both a signed compare op (cmpOp)
10000 // and an unsigned compare op (cmpOpU). it would be nice if we could
10001 // define an op class which merged both inputs and use it to type the
10002 // argument to a single rule. unfortunatelyt his fails because the
10003 // opclass does not live up to the COND_INTER interface of its
10004 // component operands. When the generic code tries to negate the
10005 // operand it ends up running the generci Machoper::negate method
10006 // which throws a ShouldNotHappen. So, we have to provide two flavours
10007 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10008 
10009 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10010   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10011 
10012   ins_cost(INSN_COST * 2);
10013   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10014 
10015   ins_encode %{
10016     __ cselw(as_Register($dst$$reg),
10017              as_Register($src2$$reg),
10018              as_Register($src1$$reg),
10019              (Assembler::Condition)$cmp$$cmpcode);
10020   %}
10021 
10022   ins_pipe(icond_reg_reg);
10023 %}
10024 
10025 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10026   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10027 
10028   ins_cost(INSN_COST * 2);
10029   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10030 
10031   ins_encode %{
10032     __ cselw(as_Register($dst$$reg),
10033              as_Register($src2$$reg),
10034              as_Register($src1$$reg),
10035              (Assembler::Condition)$cmp$$cmpcode);
10036   %}
10037 
10038   ins_pipe(icond_reg_reg);
10039 %}
10040 
10041 // special cases where one arg is zero
10042 
10043 // n.b. this is selected in preference to the rule above because it
10044 // avoids loading constant 0 into a source register
10045 
10046 // TODO
10047 // we ought only to be able to cull one of these variants as the ideal
10048 // transforms ought always to order the zero consistently (to left/right?)
10049 
10050 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10051   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10052 
10053   ins_cost(INSN_COST * 2);
10054   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10055 
10056   ins_encode %{
10057     __ cselw(as_Register($dst$$reg),
10058              as_Register($src$$reg),
10059              zr,
10060              (Assembler::Condition)$cmp$$cmpcode);
10061   %}
10062 
10063   ins_pipe(icond_reg);
10064 %}
10065 
10066 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10067   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10068 
10069   ins_cost(INSN_COST * 2);
10070   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10071 
10072   ins_encode %{
10073     __ cselw(as_Register($dst$$reg),
10074              as_Register($src$$reg),
10075              zr,
10076              (Assembler::Condition)$cmp$$cmpcode);
10077   %}
10078 
10079   ins_pipe(icond_reg);
10080 %}
10081 
10082 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10083   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10084 
10085   ins_cost(INSN_COST * 2);
10086   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10087 
10088   ins_encode %{
10089     __ cselw(as_Register($dst$$reg),
10090              zr,
10091              as_Register($src$$reg),
10092              (Assembler::Condition)$cmp$$cmpcode);
10093   %}
10094 
10095   ins_pipe(icond_reg);
10096 %}
10097 
10098 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10099   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10100 
10101   ins_cost(INSN_COST * 2);
10102   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10103 
10104   ins_encode %{
10105     __ cselw(as_Register($dst$$reg),
10106              zr,
10107              as_Register($src$$reg),
10108              (Assembler::Condition)$cmp$$cmpcode);
10109   %}
10110 
10111   ins_pipe(icond_reg);
10112 %}
10113 
10114 // special case for creating a boolean 0 or 1
10115 
10116 // n.b. this is selected in preference to the rule above because it
10117 // avoids loading constants 0 and 1 into a source register
10118 
10119 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10120   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10121 
10122   ins_cost(INSN_COST * 2);
10123   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10124 
10125   ins_encode %{
10126     // equivalently
10127     // cset(as_Register($dst$$reg),
10128     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10129     __ csincw(as_Register($dst$$reg),
10130              zr,
10131              zr,
10132              (Assembler::Condition)$cmp$$cmpcode);
10133   %}
10134 
10135   ins_pipe(icond_none);
10136 %}
10137 
10138 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10139   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10140 
10141   ins_cost(INSN_COST * 2);
10142   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10143 
10144   ins_encode %{
10145     // equivalently
10146     // cset(as_Register($dst$$reg),
10147     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10148     __ csincw(as_Register($dst$$reg),
10149              zr,
10150              zr,
10151              (Assembler::Condition)$cmp$$cmpcode);
10152   %}
10153 
10154   ins_pipe(icond_none);
10155 %}
10156 
10157 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10158   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10159 
10160   ins_cost(INSN_COST * 2);
10161   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10162 
10163   ins_encode %{
10164     __ csel(as_Register($dst$$reg),
10165             as_Register($src2$$reg),
10166             as_Register($src1$$reg),
10167             (Assembler::Condition)$cmp$$cmpcode);
10168   %}
10169 
10170   ins_pipe(icond_reg_reg);
10171 %}
10172 
10173 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10174   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10175 
10176   ins_cost(INSN_COST * 2);
10177   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10178 
10179   ins_encode %{
10180     __ csel(as_Register($dst$$reg),
10181             as_Register($src2$$reg),
10182             as_Register($src1$$reg),
10183             (Assembler::Condition)$cmp$$cmpcode);
10184   %}
10185 
10186   ins_pipe(icond_reg_reg);
10187 %}
10188 
10189 // special cases where one arg is zero
10190 
10191 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10192   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10193 
10194   ins_cost(INSN_COST * 2);
10195   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10196 
10197   ins_encode %{
10198     __ csel(as_Register($dst$$reg),
10199             zr,
10200             as_Register($src$$reg),
10201             (Assembler::Condition)$cmp$$cmpcode);
10202   %}
10203 
10204   ins_pipe(icond_reg);
10205 %}
10206 
10207 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10208   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10209 
10210   ins_cost(INSN_COST * 2);
10211   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10212 
10213   ins_encode %{
10214     __ csel(as_Register($dst$$reg),
10215             zr,
10216             as_Register($src$$reg),
10217             (Assembler::Condition)$cmp$$cmpcode);
10218   %}
10219 
10220   ins_pipe(icond_reg);
10221 %}
10222 
10223 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10224   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10225 
10226   ins_cost(INSN_COST * 2);
10227   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10228 
10229   ins_encode %{
10230     __ csel(as_Register($dst$$reg),
10231             as_Register($src$$reg),
10232             zr,
10233             (Assembler::Condition)$cmp$$cmpcode);
10234   %}
10235 
10236   ins_pipe(icond_reg);
10237 %}
10238 
10239 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10240   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10241 
10242   ins_cost(INSN_COST * 2);
10243   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10244 
10245   ins_encode %{
10246     __ csel(as_Register($dst$$reg),
10247             as_Register($src$$reg),
10248             zr,
10249             (Assembler::Condition)$cmp$$cmpcode);
10250   %}
10251 
10252   ins_pipe(icond_reg);
10253 %}
10254 
10255 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10256   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10257 
10258   ins_cost(INSN_COST * 2);
10259   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10260 
10261   ins_encode %{
10262     __ csel(as_Register($dst$$reg),
10263             as_Register($src2$$reg),
10264             as_Register($src1$$reg),
10265             (Assembler::Condition)$cmp$$cmpcode);
10266   %}
10267 
10268   ins_pipe(icond_reg_reg);
10269 %}
10270 
10271 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10272   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10273 
10274   ins_cost(INSN_COST * 2);
10275   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10276 
10277   ins_encode %{
10278     __ csel(as_Register($dst$$reg),
10279             as_Register($src2$$reg),
10280             as_Register($src1$$reg),
10281             (Assembler::Condition)$cmp$$cmpcode);
10282   %}
10283 
10284   ins_pipe(icond_reg_reg);
10285 %}
10286 
10287 // special cases where one arg is zero
10288 
10289 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10290   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10291 
10292   ins_cost(INSN_COST * 2);
10293   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10294 
10295   ins_encode %{
10296     __ csel(as_Register($dst$$reg),
10297             zr,
10298             as_Register($src$$reg),
10299             (Assembler::Condition)$cmp$$cmpcode);
10300   %}
10301 
10302   ins_pipe(icond_reg);
10303 %}
10304 
10305 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10306   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10307 
10308   ins_cost(INSN_COST * 2);
10309   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10310 
10311   ins_encode %{
10312     __ csel(as_Register($dst$$reg),
10313             zr,
10314             as_Register($src$$reg),
10315             (Assembler::Condition)$cmp$$cmpcode);
10316   %}
10317 
10318   ins_pipe(icond_reg);
10319 %}
10320 
10321 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10322   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10323 
10324   ins_cost(INSN_COST * 2);
10325   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10326 
10327   ins_encode %{
10328     __ csel(as_Register($dst$$reg),
10329             as_Register($src$$reg),
10330             zr,
10331             (Assembler::Condition)$cmp$$cmpcode);
10332   %}
10333 
10334   ins_pipe(icond_reg);
10335 %}
10336 
10337 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10338   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10339 
10340   ins_cost(INSN_COST * 2);
10341   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10342 
10343   ins_encode %{
10344     __ csel(as_Register($dst$$reg),
10345             as_Register($src$$reg),
10346             zr,
10347             (Assembler::Condition)$cmp$$cmpcode);
10348   %}
10349 
10350   ins_pipe(icond_reg);
10351 %}
10352 
10353 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10354   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10355 
10356   ins_cost(INSN_COST * 2);
10357   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10358 
10359   ins_encode %{
10360     __ cselw(as_Register($dst$$reg),
10361              as_Register($src2$$reg),
10362              as_Register($src1$$reg),
10363              (Assembler::Condition)$cmp$$cmpcode);
10364   %}
10365 
10366   ins_pipe(icond_reg_reg);
10367 %}
10368 
10369 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10370   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10371 
10372   ins_cost(INSN_COST * 2);
10373   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10374 
10375   ins_encode %{
10376     __ cselw(as_Register($dst$$reg),
10377              as_Register($src2$$reg),
10378              as_Register($src1$$reg),
10379              (Assembler::Condition)$cmp$$cmpcode);
10380   %}
10381 
10382   ins_pipe(icond_reg_reg);
10383 %}
10384 
10385 // special cases where one arg is zero
10386 
10387 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10388   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10389 
10390   ins_cost(INSN_COST * 2);
10391   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10392 
10393   ins_encode %{
10394     __ cselw(as_Register($dst$$reg),
10395              zr,
10396              as_Register($src$$reg),
10397              (Assembler::Condition)$cmp$$cmpcode);
10398   %}
10399 
10400   ins_pipe(icond_reg);
10401 %}
10402 
10403 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10404   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10405 
10406   ins_cost(INSN_COST * 2);
10407   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10408 
10409   ins_encode %{
10410     __ cselw(as_Register($dst$$reg),
10411              zr,
10412              as_Register($src$$reg),
10413              (Assembler::Condition)$cmp$$cmpcode);
10414   %}
10415 
10416   ins_pipe(icond_reg);
10417 %}
10418 
10419 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10420   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10421 
10422   ins_cost(INSN_COST * 2);
10423   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10424 
10425   ins_encode %{
10426     __ cselw(as_Register($dst$$reg),
10427              as_Register($src$$reg),
10428              zr,
10429              (Assembler::Condition)$cmp$$cmpcode);
10430   %}
10431 
10432   ins_pipe(icond_reg);
10433 %}
10434 
10435 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10436   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10437 
10438   ins_cost(INSN_COST * 2);
10439   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10440 
10441   ins_encode %{
10442     __ cselw(as_Register($dst$$reg),
10443              as_Register($src$$reg),
10444              zr,
10445              (Assembler::Condition)$cmp$$cmpcode);
10446   %}
10447 
10448   ins_pipe(icond_reg);
10449 %}
10450 
10451 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10452 %{
10453   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10454 
10455   ins_cost(INSN_COST * 3);
10456 
10457   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10458   ins_encode %{
10459     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10460     __ fcsels(as_FloatRegister($dst$$reg),
10461               as_FloatRegister($src2$$reg),
10462               as_FloatRegister($src1$$reg),
10463               cond);
10464   %}
10465 
10466   ins_pipe(fp_cond_reg_reg_s);
10467 %}
10468 
10469 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10470 %{
10471   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10472 
10473   ins_cost(INSN_COST * 3);
10474 
10475   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10476   ins_encode %{
10477     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10478     __ fcsels(as_FloatRegister($dst$$reg),
10479               as_FloatRegister($src2$$reg),
10480               as_FloatRegister($src1$$reg),
10481               cond);
10482   %}
10483 
10484   ins_pipe(fp_cond_reg_reg_s);
10485 %}
10486 
10487 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10488 %{
10489   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10490 
10491   ins_cost(INSN_COST * 3);
10492 
10493   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10494   ins_encode %{
10495     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10496     __ fcseld(as_FloatRegister($dst$$reg),
10497               as_FloatRegister($src2$$reg),
10498               as_FloatRegister($src1$$reg),
10499               cond);
10500   %}
10501 
10502   ins_pipe(fp_cond_reg_reg_d);
10503 %}
10504 
10505 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10506 %{
10507   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10508 
10509   ins_cost(INSN_COST * 3);
10510 
10511   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10512   ins_encode %{
10513     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10514     __ fcseld(as_FloatRegister($dst$$reg),
10515               as_FloatRegister($src2$$reg),
10516               as_FloatRegister($src1$$reg),
10517               cond);
10518   %}
10519 
10520   ins_pipe(fp_cond_reg_reg_d);
10521 %}
10522 
10523 // ============================================================================
10524 // Arithmetic Instructions
10525 //
10526 
10527 // Integer Addition
10528 
10529 // TODO
10530 // these currently employ operations which do not set CR and hence are
10531 // not flagged as killing CR but we would like to isolate the cases
10532 // where we want to set flags from those where we don't. need to work
10533 // out how to do that.
10534 
10535 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10536   match(Set dst (AddI src1 src2));
10537 
10538   ins_cost(INSN_COST);
10539   format %{ "addw  $dst, $src1, $src2" %}
10540 
10541   ins_encode %{
10542     __ addw(as_Register($dst$$reg),
10543             as_Register($src1$$reg),
10544             as_Register($src2$$reg));
10545   %}
10546 
10547   ins_pipe(ialu_reg_reg);
10548 %}
10549 
10550 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10551   match(Set dst (AddI src1 src2));
10552 
10553   ins_cost(INSN_COST);
10554   format %{ "addw $dst, $src1, $src2" %}
10555 
10556   // use opcode to indicate that this is an add not a sub
10557   opcode(0x0);
10558 
10559   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10560 
10561   ins_pipe(ialu_reg_imm);
10562 %}
10563 
10564 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10565   match(Set dst (AddI (ConvL2I src1) src2));
10566 
10567   ins_cost(INSN_COST);
10568   format %{ "addw $dst, $src1, $src2" %}
10569 
10570   // use opcode to indicate that this is an add not a sub
10571   opcode(0x0);
10572 
10573   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10574 
10575   ins_pipe(ialu_reg_imm);
10576 %}
10577 
10578 // Pointer Addition
10579 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10580   match(Set dst (AddP src1 src2));
10581 
10582   ins_cost(INSN_COST);
10583   format %{ "add $dst, $src1, $src2\t# ptr" %}
10584 
10585   ins_encode %{
10586     __ add(as_Register($dst$$reg),
10587            as_Register($src1$$reg),
10588            as_Register($src2$$reg));
10589   %}
10590 
10591   ins_pipe(ialu_reg_reg);
10592 %}
10593 
10594 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10595   match(Set dst (AddP src1 (ConvI2L src2)));
10596 
10597   ins_cost(1.9 * INSN_COST);
10598   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10599 
10600   ins_encode %{
10601     __ add(as_Register($dst$$reg),
10602            as_Register($src1$$reg),
10603            as_Register($src2$$reg), ext::sxtw);
10604   %}
10605 
10606   ins_pipe(ialu_reg_reg);
10607 %}
10608 
10609 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10610   match(Set dst (AddP src1 (LShiftL src2 scale)));
10611 
10612   ins_cost(1.9 * INSN_COST);
10613   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10614 
10615   ins_encode %{
10616     __ lea(as_Register($dst$$reg),
10617            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10618                    Address::lsl($scale$$constant)));
10619   %}
10620 
10621   ins_pipe(ialu_reg_reg_shift);
10622 %}
10623 
10624 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10625   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10626 
10627   ins_cost(1.9 * INSN_COST);
10628   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10629 
10630   ins_encode %{
10631     __ lea(as_Register($dst$$reg),
10632            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10633                    Address::sxtw($scale$$constant)));
10634   %}
10635 
10636   ins_pipe(ialu_reg_reg_shift);
10637 %}
10638 
10639 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10640   match(Set dst (LShiftL (ConvI2L src) scale));
10641 
10642   ins_cost(INSN_COST);
10643   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10644 
10645   ins_encode %{
10646     __ sbfiz(as_Register($dst$$reg),
10647           as_Register($src$$reg),
10648           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10649   %}
10650 
10651   ins_pipe(ialu_reg_shift);
10652 %}
10653 
10654 // Pointer Immediate Addition
10655 // n.b. this needs to be more expensive than using an indirect memory
10656 // operand
10657 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10658   match(Set dst (AddP src1 src2));
10659 
10660   ins_cost(INSN_COST);
10661   format %{ "add $dst, $src1, $src2\t# ptr" %}
10662 
10663   // use opcode to indicate that this is an add not a sub
10664   opcode(0x0);
10665 
10666   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10667 
10668   ins_pipe(ialu_reg_imm);
10669 %}
10670 
10671 // Long Addition
10672 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10673 
10674   match(Set dst (AddL src1 src2));
10675 
10676   ins_cost(INSN_COST);
10677   format %{ "add  $dst, $src1, $src2" %}
10678 
10679   ins_encode %{
10680     __ add(as_Register($dst$$reg),
10681            as_Register($src1$$reg),
10682            as_Register($src2$$reg));
10683   %}
10684 
10685   ins_pipe(ialu_reg_reg);
10686 %}
10687 
10688 // No constant pool entries requiredLong Immediate Addition.
10689 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10690   match(Set dst (AddL src1 src2));
10691 
10692   ins_cost(INSN_COST);
10693   format %{ "add $dst, $src1, $src2" %}
10694 
10695   // use opcode to indicate that this is an add not a sub
10696   opcode(0x0);
10697 
10698   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10699 
10700   ins_pipe(ialu_reg_imm);
10701 %}
10702 
10703 // Integer Subtraction
10704 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10705   match(Set dst (SubI src1 src2));
10706 
10707   ins_cost(INSN_COST);
10708   format %{ "subw  $dst, $src1, $src2" %}
10709 
10710   ins_encode %{
10711     __ subw(as_Register($dst$$reg),
10712             as_Register($src1$$reg),
10713             as_Register($src2$$reg));
10714   %}
10715 
10716   ins_pipe(ialu_reg_reg);
10717 %}
10718 
10719 // Immediate Subtraction
10720 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10721   match(Set dst (SubI src1 src2));
10722 
10723   ins_cost(INSN_COST);
10724   format %{ "subw $dst, $src1, $src2" %}
10725 
10726   // use opcode to indicate that this is a sub not an add
10727   opcode(0x1);
10728 
10729   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10730 
10731   ins_pipe(ialu_reg_imm);
10732 %}
10733 
10734 // Long Subtraction
10735 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10736 
10737   match(Set dst (SubL src1 src2));
10738 
10739   ins_cost(INSN_COST);
10740   format %{ "sub  $dst, $src1, $src2" %}
10741 
10742   ins_encode %{
10743     __ sub(as_Register($dst$$reg),
10744            as_Register($src1$$reg),
10745            as_Register($src2$$reg));
10746   %}
10747 
10748   ins_pipe(ialu_reg_reg);
10749 %}
10750 
10751 // No constant pool entries requiredLong Immediate Subtraction.
10752 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10753   match(Set dst (SubL src1 src2));
10754 
10755   ins_cost(INSN_COST);
10756   format %{ "sub$dst, $src1, $src2" %}
10757 
10758   // use opcode to indicate that this is a sub not an add
10759   opcode(0x1);
10760 
10761   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10762 
10763   ins_pipe(ialu_reg_imm);
10764 %}
10765 
10766 // Integer Negation (special case for sub)
10767 
10768 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10769   match(Set dst (SubI zero src));
10770 
10771   ins_cost(INSN_COST);
10772   format %{ "negw $dst, $src\t# int" %}
10773 
10774   ins_encode %{
10775     __ negw(as_Register($dst$$reg),
10776             as_Register($src$$reg));
10777   %}
10778 
10779   ins_pipe(ialu_reg);
10780 %}
10781 
10782 // Long Negation
10783 
10784 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10785   match(Set dst (SubL zero src));
10786 
10787   ins_cost(INSN_COST);
10788   format %{ "neg $dst, $src\t# long" %}
10789 
10790   ins_encode %{
10791     __ neg(as_Register($dst$$reg),
10792            as_Register($src$$reg));
10793   %}
10794 
10795   ins_pipe(ialu_reg);
10796 %}
10797 
10798 // Integer Multiply
10799 
10800 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10801   match(Set dst (MulI src1 src2));
10802 
10803   ins_cost(INSN_COST * 3);
10804   format %{ "mulw  $dst, $src1, $src2" %}
10805 
10806   ins_encode %{
10807     __ mulw(as_Register($dst$$reg),
10808             as_Register($src1$$reg),
10809             as_Register($src2$$reg));
10810   %}
10811 
10812   ins_pipe(imul_reg_reg);
10813 %}
10814 
10815 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10816   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10817 
10818   ins_cost(INSN_COST * 3);
10819   format %{ "smull  $dst, $src1, $src2" %}
10820 
10821   ins_encode %{
10822     __ smull(as_Register($dst$$reg),
10823              as_Register($src1$$reg),
10824              as_Register($src2$$reg));
10825   %}
10826 
10827   ins_pipe(imul_reg_reg);
10828 %}
10829 
10830 // Long Multiply
10831 
10832 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10833   match(Set dst (MulL src1 src2));
10834 
10835   ins_cost(INSN_COST * 5);
10836   format %{ "mul  $dst, $src1, $src2" %}
10837 
10838   ins_encode %{
10839     __ mul(as_Register($dst$$reg),
10840            as_Register($src1$$reg),
10841            as_Register($src2$$reg));
10842   %}
10843 
10844   ins_pipe(lmul_reg_reg);
10845 %}
10846 
10847 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10848 %{
10849   match(Set dst (MulHiL src1 src2));
10850 
10851   ins_cost(INSN_COST * 7);
10852   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10853 
10854   ins_encode %{
10855     __ smulh(as_Register($dst$$reg),
10856              as_Register($src1$$reg),
10857              as_Register($src2$$reg));
10858   %}
10859 
10860   ins_pipe(lmul_reg_reg);
10861 %}
10862 
10863 // Combined Integer Multiply & Add/Sub
10864 
10865 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10866   match(Set dst (AddI src3 (MulI src1 src2)));
10867 
10868   ins_cost(INSN_COST * 3);
10869   format %{ "madd  $dst, $src1, $src2, $src3" %}
10870 
10871   ins_encode %{
10872     __ maddw(as_Register($dst$$reg),
10873              as_Register($src1$$reg),
10874              as_Register($src2$$reg),
10875              as_Register($src3$$reg));
10876   %}
10877 
10878   ins_pipe(imac_reg_reg);
10879 %}
10880 
10881 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10882   match(Set dst (SubI src3 (MulI src1 src2)));
10883 
10884   ins_cost(INSN_COST * 3);
10885   format %{ "msub  $dst, $src1, $src2, $src3" %}
10886 
10887   ins_encode %{
10888     __ msubw(as_Register($dst$$reg),
10889              as_Register($src1$$reg),
10890              as_Register($src2$$reg),
10891              as_Register($src3$$reg));
10892   %}
10893 
10894   ins_pipe(imac_reg_reg);
10895 %}
10896 
10897 // Combined Integer Multiply & Neg
10898 
10899 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10900   match(Set dst (MulI (SubI zero src1) src2));
10901 
10902   ins_cost(INSN_COST * 3);
10903   format %{ "mneg  $dst, $src1, $src2" %}
10904 
10905   ins_encode %{
10906     __ mnegw(as_Register($dst$$reg),
10907              as_Register($src1$$reg),
10908              as_Register($src2$$reg));
10909   %}
10910 
10911   ins_pipe(imac_reg_reg);
10912 %}
10913 
10914 // Combined Long Multiply & Add/Sub
10915 
10916 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10917   match(Set dst (AddL src3 (MulL src1 src2)));
10918 
10919   ins_cost(INSN_COST * 5);
10920   format %{ "madd  $dst, $src1, $src2, $src3" %}
10921 
10922   ins_encode %{
10923     __ madd(as_Register($dst$$reg),
10924             as_Register($src1$$reg),
10925             as_Register($src2$$reg),
10926             as_Register($src3$$reg));
10927   %}
10928 
10929   ins_pipe(lmac_reg_reg);
10930 %}
10931 
10932 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10933   match(Set dst (SubL src3 (MulL src1 src2)));
10934 
10935   ins_cost(INSN_COST * 5);
10936   format %{ "msub  $dst, $src1, $src2, $src3" %}
10937 
10938   ins_encode %{
10939     __ msub(as_Register($dst$$reg),
10940             as_Register($src1$$reg),
10941             as_Register($src2$$reg),
10942             as_Register($src3$$reg));
10943   %}
10944 
10945   ins_pipe(lmac_reg_reg);
10946 %}
10947 
10948 // Combined Long Multiply & Neg
10949 
10950 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10951   match(Set dst (MulL (SubL zero src1) src2));
10952 
10953   ins_cost(INSN_COST * 5);
10954   format %{ "mneg  $dst, $src1, $src2" %}
10955 
10956   ins_encode %{
10957     __ mneg(as_Register($dst$$reg),
10958             as_Register($src1$$reg),
10959             as_Register($src2$$reg));
10960   %}
10961 
10962   ins_pipe(lmac_reg_reg);
10963 %}
10964 
10965 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10966 
10967 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10968   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10969 
10970   ins_cost(INSN_COST * 3);
10971   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10972 
10973   ins_encode %{
10974     __ smaddl(as_Register($dst$$reg),
10975               as_Register($src1$$reg),
10976               as_Register($src2$$reg),
10977               as_Register($src3$$reg));
10978   %}
10979 
10980   ins_pipe(imac_reg_reg);
10981 %}
10982 
10983 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10984   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10985 
10986   ins_cost(INSN_COST * 3);
10987   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10988 
10989   ins_encode %{
10990     __ smsubl(as_Register($dst$$reg),
10991               as_Register($src1$$reg),
10992               as_Register($src2$$reg),
10993               as_Register($src3$$reg));
10994   %}
10995 
10996   ins_pipe(imac_reg_reg);
10997 %}
10998 
10999 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11000   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11001 
11002   ins_cost(INSN_COST * 3);
11003   format %{ "smnegl  $dst, $src1, $src2" %}
11004 
11005   ins_encode %{
11006     __ smnegl(as_Register($dst$$reg),
11007               as_Register($src1$$reg),
11008               as_Register($src2$$reg));
11009   %}
11010 
11011   ins_pipe(imac_reg_reg);
11012 %}
11013 
11014 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11015 
11016 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11017   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11018 
11019   ins_cost(INSN_COST * 5);
11020   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11021             "maddw $dst, $src3, $src4, rscratch1" %}
11022 
11023   ins_encode %{
11024     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11025     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11026 
11027   ins_pipe(imac_reg_reg);
11028 %}
11029 
11030 // Integer Divide
11031 
11032 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11033   match(Set dst (DivI src1 src2));
11034 
11035   ins_cost(INSN_COST * 19);
11036   format %{ "sdivw  $dst, $src1, $src2" %}
11037 
11038   ins_encode(aarch64_enc_divw(dst, src1, src2));
11039   ins_pipe(idiv_reg_reg);
11040 %}
11041 
11042 // Long Divide
11043 
11044 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11045   match(Set dst (DivL src1 src2));
11046 
11047   ins_cost(INSN_COST * 35);
11048   format %{ "sdiv   $dst, $src1, $src2" %}
11049 
11050   ins_encode(aarch64_enc_div(dst, src1, src2));
11051   ins_pipe(ldiv_reg_reg);
11052 %}
11053 
11054 // Integer Remainder
11055 
11056 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11057   match(Set dst (ModI src1 src2));
11058 
11059   ins_cost(INSN_COST * 22);
11060   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11061             "msubw($dst, rscratch1, $src2, $src1" %}
11062 
11063   ins_encode(aarch64_enc_modw(dst, src1, src2));
11064   ins_pipe(idiv_reg_reg);
11065 %}
11066 
11067 // Long Remainder
11068 
11069 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11070   match(Set dst (ModL src1 src2));
11071 
11072   ins_cost(INSN_COST * 38);
11073   format %{ "sdiv   rscratch1, $src1, $src2\n"
11074             "msub($dst, rscratch1, $src2, $src1" %}
11075 
11076   ins_encode(aarch64_enc_mod(dst, src1, src2));
11077   ins_pipe(ldiv_reg_reg);
11078 %}
11079 
11080 // Integer Shifts
11081 
11082 // Shift Left Register
11083 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11084   match(Set dst (LShiftI src1 src2));
11085 
11086   ins_cost(INSN_COST * 2);
11087   format %{ "lslvw  $dst, $src1, $src2" %}
11088 
11089   ins_encode %{
11090     __ lslvw(as_Register($dst$$reg),
11091              as_Register($src1$$reg),
11092              as_Register($src2$$reg));
11093   %}
11094 
11095   ins_pipe(ialu_reg_reg_vshift);
11096 %}
11097 
11098 // Shift Left Immediate
11099 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11100   match(Set dst (LShiftI src1 src2));
11101 
11102   ins_cost(INSN_COST);
11103   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11104 
11105   ins_encode %{
11106     __ lslw(as_Register($dst$$reg),
11107             as_Register($src1$$reg),
11108             $src2$$constant & 0x1f);
11109   %}
11110 
11111   ins_pipe(ialu_reg_shift);
11112 %}
11113 
11114 // Shift Right Logical Register
11115 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11116   match(Set dst (URShiftI src1 src2));
11117 
11118   ins_cost(INSN_COST * 2);
11119   format %{ "lsrvw  $dst, $src1, $src2" %}
11120 
11121   ins_encode %{
11122     __ lsrvw(as_Register($dst$$reg),
11123              as_Register($src1$$reg),
11124              as_Register($src2$$reg));
11125   %}
11126 
11127   ins_pipe(ialu_reg_reg_vshift);
11128 %}
11129 
11130 // Shift Right Logical Immediate
11131 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11132   match(Set dst (URShiftI src1 src2));
11133 
11134   ins_cost(INSN_COST);
11135   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11136 
11137   ins_encode %{
11138     __ lsrw(as_Register($dst$$reg),
11139             as_Register($src1$$reg),
11140             $src2$$constant & 0x1f);
11141   %}
11142 
11143   ins_pipe(ialu_reg_shift);
11144 %}
11145 
11146 // Shift Right Arithmetic Register
11147 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11148   match(Set dst (RShiftI src1 src2));
11149 
11150   ins_cost(INSN_COST * 2);
11151   format %{ "asrvw  $dst, $src1, $src2" %}
11152 
11153   ins_encode %{
11154     __ asrvw(as_Register($dst$$reg),
11155              as_Register($src1$$reg),
11156              as_Register($src2$$reg));
11157   %}
11158 
11159   ins_pipe(ialu_reg_reg_vshift);
11160 %}
11161 
11162 // Shift Right Arithmetic Immediate
11163 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11164   match(Set dst (RShiftI src1 src2));
11165 
11166   ins_cost(INSN_COST);
11167   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11168 
11169   ins_encode %{
11170     __ asrw(as_Register($dst$$reg),
11171             as_Register($src1$$reg),
11172             $src2$$constant & 0x1f);
11173   %}
11174 
11175   ins_pipe(ialu_reg_shift);
11176 %}
11177 
11178 // Combined Int Mask and Right Shift (using UBFM)
11179 // TODO
11180 
11181 // Long Shifts
11182 
11183 // Shift Left Register
11184 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11185   match(Set dst (LShiftL src1 src2));
11186 
11187   ins_cost(INSN_COST * 2);
11188   format %{ "lslv  $dst, $src1, $src2" %}
11189 
11190   ins_encode %{
11191     __ lslv(as_Register($dst$$reg),
11192             as_Register($src1$$reg),
11193             as_Register($src2$$reg));
11194   %}
11195 
11196   ins_pipe(ialu_reg_reg_vshift);
11197 %}
11198 
11199 // Shift Left Immediate
11200 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11201   match(Set dst (LShiftL src1 src2));
11202 
11203   ins_cost(INSN_COST);
11204   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11205 
11206   ins_encode %{
11207     __ lsl(as_Register($dst$$reg),
11208             as_Register($src1$$reg),
11209             $src2$$constant & 0x3f);
11210   %}
11211 
11212   ins_pipe(ialu_reg_shift);
11213 %}
11214 
11215 // Shift Right Logical Register
11216 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11217   match(Set dst (URShiftL src1 src2));
11218 
11219   ins_cost(INSN_COST * 2);
11220   format %{ "lsrv  $dst, $src1, $src2" %}
11221 
11222   ins_encode %{
11223     __ lsrv(as_Register($dst$$reg),
11224             as_Register($src1$$reg),
11225             as_Register($src2$$reg));
11226   %}
11227 
11228   ins_pipe(ialu_reg_reg_vshift);
11229 %}
11230 
11231 // Shift Right Logical Immediate
11232 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11233   match(Set dst (URShiftL src1 src2));
11234 
11235   ins_cost(INSN_COST);
11236   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11237 
11238   ins_encode %{
11239     __ lsr(as_Register($dst$$reg),
11240            as_Register($src1$$reg),
11241            $src2$$constant & 0x3f);
11242   %}
11243 
11244   ins_pipe(ialu_reg_shift);
11245 %}
11246 
11247 // A special-case pattern for card table stores.
11248 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11249   match(Set dst (URShiftL (CastP2X src1) src2));
11250 
11251   ins_cost(INSN_COST);
11252   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11253 
11254   ins_encode %{
11255     __ lsr(as_Register($dst$$reg),
11256            as_Register($src1$$reg),
11257            $src2$$constant & 0x3f);
11258   %}
11259 
11260   ins_pipe(ialu_reg_shift);
11261 %}
11262 
11263 // Shift Right Arithmetic Register
11264 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11265   match(Set dst (RShiftL src1 src2));
11266 
11267   ins_cost(INSN_COST * 2);
11268   format %{ "asrv  $dst, $src1, $src2" %}
11269 
11270   ins_encode %{
11271     __ asrv(as_Register($dst$$reg),
11272             as_Register($src1$$reg),
11273             as_Register($src2$$reg));
11274   %}
11275 
11276   ins_pipe(ialu_reg_reg_vshift);
11277 %}
11278 
11279 // Shift Right Arithmetic Immediate
11280 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11281   match(Set dst (RShiftL src1 src2));
11282 
11283   ins_cost(INSN_COST);
11284   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11285 
11286   ins_encode %{
11287     __ asr(as_Register($dst$$reg),
11288            as_Register($src1$$reg),
11289            $src2$$constant & 0x3f);
11290   %}
11291 
11292   ins_pipe(ialu_reg_shift);
11293 %}
11294 
11295 // BEGIN This section of the file is automatically generated. Do not edit --------------
11296 // This section is generated from aarch64_ad.m4
11297 
11298 
11299 // This pattern is automatically generated from aarch64_ad.m4
11300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11301 instruct regL_not_reg(iRegLNoSp dst,
11302                          iRegL src1, immL_M1 m1,
11303                          rFlagsReg cr) %{
11304   match(Set dst (XorL src1 m1));
11305   ins_cost(INSN_COST);
11306   format %{ "eon  $dst, $src1, zr" %}
11307 
11308   ins_encode %{
11309     __ eon(as_Register($dst$$reg),
11310               as_Register($src1$$reg),
11311               zr,
11312               Assembler::LSL, 0);
11313   %}
11314 
11315   ins_pipe(ialu_reg);
11316 %}
11317 
11318 // This pattern is automatically generated from aarch64_ad.m4
11319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11320 instruct regI_not_reg(iRegINoSp dst,
11321                          iRegIorL2I src1, immI_M1 m1,
11322                          rFlagsReg cr) %{
11323   match(Set dst (XorI src1 m1));
11324   ins_cost(INSN_COST);
11325   format %{ "eonw  $dst, $src1, zr" %}
11326 
11327   ins_encode %{
11328     __ eonw(as_Register($dst$$reg),
11329               as_Register($src1$$reg),
11330               zr,
11331               Assembler::LSL, 0);
11332   %}
11333 
11334   ins_pipe(ialu_reg);
11335 %}
11336 
11337 // This pattern is automatically generated from aarch64_ad.m4
11338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11339 instruct AndI_reg_not_reg(iRegINoSp dst,
11340                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11341   match(Set dst (AndI src1 (XorI src2 m1)));
11342   ins_cost(INSN_COST);
11343   format %{ "bicw  $dst, $src1, $src2" %}
11344 
11345   ins_encode %{
11346     __ bicw(as_Register($dst$$reg),
11347               as_Register($src1$$reg),
11348               as_Register($src2$$reg),
11349               Assembler::LSL, 0);
11350   %}
11351 
11352   ins_pipe(ialu_reg_reg);
11353 %}
11354 
11355 // This pattern is automatically generated from aarch64_ad.m4
11356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11357 instruct AndL_reg_not_reg(iRegLNoSp dst,
11358                          iRegL src1, iRegL src2, immL_M1 m1) %{
11359   match(Set dst (AndL src1 (XorL src2 m1)));
11360   ins_cost(INSN_COST);
11361   format %{ "bic  $dst, $src1, $src2" %}
11362 
11363   ins_encode %{
11364     __ bic(as_Register($dst$$reg),
11365               as_Register($src1$$reg),
11366               as_Register($src2$$reg),
11367               Assembler::LSL, 0);
11368   %}
11369 
11370   ins_pipe(ialu_reg_reg);
11371 %}
11372 
11373 // This pattern is automatically generated from aarch64_ad.m4
11374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11375 instruct OrI_reg_not_reg(iRegINoSp dst,
11376                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11377   match(Set dst (OrI src1 (XorI src2 m1)));
11378   ins_cost(INSN_COST);
11379   format %{ "ornw  $dst, $src1, $src2" %}
11380 
11381   ins_encode %{
11382     __ ornw(as_Register($dst$$reg),
11383               as_Register($src1$$reg),
11384               as_Register($src2$$reg),
11385               Assembler::LSL, 0);
11386   %}
11387 
11388   ins_pipe(ialu_reg_reg);
11389 %}
11390 
11391 // This pattern is automatically generated from aarch64_ad.m4
11392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11393 instruct OrL_reg_not_reg(iRegLNoSp dst,
11394                          iRegL src1, iRegL src2, immL_M1 m1) %{
11395   match(Set dst (OrL src1 (XorL src2 m1)));
11396   ins_cost(INSN_COST);
11397   format %{ "orn  $dst, $src1, $src2" %}
11398 
11399   ins_encode %{
11400     __ orn(as_Register($dst$$reg),
11401               as_Register($src1$$reg),
11402               as_Register($src2$$reg),
11403               Assembler::LSL, 0);
11404   %}
11405 
11406   ins_pipe(ialu_reg_reg);
11407 %}
11408 
11409 // This pattern is automatically generated from aarch64_ad.m4
11410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11411 instruct XorI_reg_not_reg(iRegINoSp dst,
11412                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11413   match(Set dst (XorI m1 (XorI src2 src1)));
11414   ins_cost(INSN_COST);
11415   format %{ "eonw  $dst, $src1, $src2" %}
11416 
11417   ins_encode %{
11418     __ eonw(as_Register($dst$$reg),
11419               as_Register($src1$$reg),
11420               as_Register($src2$$reg),
11421               Assembler::LSL, 0);
11422   %}
11423 
11424   ins_pipe(ialu_reg_reg);
11425 %}
11426 
11427 // This pattern is automatically generated from aarch64_ad.m4
11428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11429 instruct XorL_reg_not_reg(iRegLNoSp dst,
11430                          iRegL src1, iRegL src2, immL_M1 m1) %{
11431   match(Set dst (XorL m1 (XorL src2 src1)));
11432   ins_cost(INSN_COST);
11433   format %{ "eon  $dst, $src1, $src2" %}
11434 
11435   ins_encode %{
11436     __ eon(as_Register($dst$$reg),
11437               as_Register($src1$$reg),
11438               as_Register($src2$$reg),
11439               Assembler::LSL, 0);
11440   %}
11441 
11442   ins_pipe(ialu_reg_reg);
11443 %}
11444 
11445 // This pattern is automatically generated from aarch64_ad.m4
11446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11447 // val & (-1 ^ (val >>> shift)) ==> bicw
11448 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11449                          iRegIorL2I src1, iRegIorL2I src2,
11450                          immI src3, immI_M1 src4) %{
11451   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11452   ins_cost(1.9 * INSN_COST);
11453   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11454 
11455   ins_encode %{
11456     __ bicw(as_Register($dst$$reg),
11457               as_Register($src1$$reg),
11458               as_Register($src2$$reg),
11459               Assembler::LSR,
11460               $src3$$constant & 0x1f);
11461   %}
11462 
11463   ins_pipe(ialu_reg_reg_shift);
11464 %}
11465 
11466 // This pattern is automatically generated from aarch64_ad.m4
11467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11468 // val & (-1 ^ (val >>> shift)) ==> bic
11469 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11470                          iRegL src1, iRegL src2,
11471                          immI src3, immL_M1 src4) %{
11472   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11473   ins_cost(1.9 * INSN_COST);
11474   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11475 
11476   ins_encode %{
11477     __ bic(as_Register($dst$$reg),
11478               as_Register($src1$$reg),
11479               as_Register($src2$$reg),
11480               Assembler::LSR,
11481               $src3$$constant & 0x3f);
11482   %}
11483 
11484   ins_pipe(ialu_reg_reg_shift);
11485 %}
11486 
11487 // This pattern is automatically generated from aarch64_ad.m4
11488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11489 // val & (-1 ^ (val >> shift)) ==> bicw
11490 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11491                          iRegIorL2I src1, iRegIorL2I src2,
11492                          immI src3, immI_M1 src4) %{
11493   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11494   ins_cost(1.9 * INSN_COST);
11495   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11496 
11497   ins_encode %{
11498     __ bicw(as_Register($dst$$reg),
11499               as_Register($src1$$reg),
11500               as_Register($src2$$reg),
11501               Assembler::ASR,
11502               $src3$$constant & 0x1f);
11503   %}
11504 
11505   ins_pipe(ialu_reg_reg_shift);
11506 %}
11507 
11508 // This pattern is automatically generated from aarch64_ad.m4
11509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11510 // val & (-1 ^ (val >> shift)) ==> bic
11511 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11512                          iRegL src1, iRegL src2,
11513                          immI src3, immL_M1 src4) %{
11514   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11515   ins_cost(1.9 * INSN_COST);
11516   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11517 
11518   ins_encode %{
11519     __ bic(as_Register($dst$$reg),
11520               as_Register($src1$$reg),
11521               as_Register($src2$$reg),
11522               Assembler::ASR,
11523               $src3$$constant & 0x3f);
11524   %}
11525 
11526   ins_pipe(ialu_reg_reg_shift);
11527 %}
11528 
11529 // This pattern is automatically generated from aarch64_ad.m4
11530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11531 // val & (-1 ^ (val ror shift)) ==> bicw
11532 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11533                          iRegIorL2I src1, iRegIorL2I src2,
11534                          immI src3, immI_M1 src4) %{
11535   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11536   ins_cost(1.9 * INSN_COST);
11537   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11538 
11539   ins_encode %{
11540     __ bicw(as_Register($dst$$reg),
11541               as_Register($src1$$reg),
11542               as_Register($src2$$reg),
11543               Assembler::ROR,
11544               $src3$$constant & 0x1f);
11545   %}
11546 
11547   ins_pipe(ialu_reg_reg_shift);
11548 %}
11549 
11550 // This pattern is automatically generated from aarch64_ad.m4
11551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11552 // val & (-1 ^ (val ror shift)) ==> bic
11553 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11554                          iRegL src1, iRegL src2,
11555                          immI src3, immL_M1 src4) %{
11556   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11557   ins_cost(1.9 * INSN_COST);
11558   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11559 
11560   ins_encode %{
11561     __ bic(as_Register($dst$$reg),
11562               as_Register($src1$$reg),
11563               as_Register($src2$$reg),
11564               Assembler::ROR,
11565               $src3$$constant & 0x3f);
11566   %}
11567 
11568   ins_pipe(ialu_reg_reg_shift);
11569 %}
11570 
11571 // This pattern is automatically generated from aarch64_ad.m4
11572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11573 // val & (-1 ^ (val << shift)) ==> bicw
11574 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11575                          iRegIorL2I src1, iRegIorL2I src2,
11576                          immI src3, immI_M1 src4) %{
11577   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11578   ins_cost(1.9 * INSN_COST);
11579   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11580 
11581   ins_encode %{
11582     __ bicw(as_Register($dst$$reg),
11583               as_Register($src1$$reg),
11584               as_Register($src2$$reg),
11585               Assembler::LSL,
11586               $src3$$constant & 0x1f);
11587   %}
11588 
11589   ins_pipe(ialu_reg_reg_shift);
11590 %}
11591 
11592 // This pattern is automatically generated from aarch64_ad.m4
11593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11594 // val & (-1 ^ (val << shift)) ==> bic
11595 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11596                          iRegL src1, iRegL src2,
11597                          immI src3, immL_M1 src4) %{
11598   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11599   ins_cost(1.9 * INSN_COST);
11600   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11601 
11602   ins_encode %{
11603     __ bic(as_Register($dst$$reg),
11604               as_Register($src1$$reg),
11605               as_Register($src2$$reg),
11606               Assembler::LSL,
11607               $src3$$constant & 0x3f);
11608   %}
11609 
11610   ins_pipe(ialu_reg_reg_shift);
11611 %}
11612 
11613 // This pattern is automatically generated from aarch64_ad.m4
11614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11615 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11616 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11617                          iRegIorL2I src1, iRegIorL2I src2,
11618                          immI src3, immI_M1 src4) %{
11619   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11620   ins_cost(1.9 * INSN_COST);
11621   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11622 
11623   ins_encode %{
11624     __ eonw(as_Register($dst$$reg),
11625               as_Register($src1$$reg),
11626               as_Register($src2$$reg),
11627               Assembler::LSR,
11628               $src3$$constant & 0x1f);
11629   %}
11630 
11631   ins_pipe(ialu_reg_reg_shift);
11632 %}
11633 
11634 // This pattern is automatically generated from aarch64_ad.m4
11635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11636 // val ^ (-1 ^ (val >>> shift)) ==> eon
11637 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11638                          iRegL src1, iRegL src2,
11639                          immI src3, immL_M1 src4) %{
11640   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11641   ins_cost(1.9 * INSN_COST);
11642   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11643 
11644   ins_encode %{
11645     __ eon(as_Register($dst$$reg),
11646               as_Register($src1$$reg),
11647               as_Register($src2$$reg),
11648               Assembler::LSR,
11649               $src3$$constant & 0x3f);
11650   %}
11651 
11652   ins_pipe(ialu_reg_reg_shift);
11653 %}
11654 
11655 // This pattern is automatically generated from aarch64_ad.m4
11656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11657 // val ^ (-1 ^ (val >> shift)) ==> eonw
11658 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11659                          iRegIorL2I src1, iRegIorL2I src2,
11660                          immI src3, immI_M1 src4) %{
11661   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11662   ins_cost(1.9 * INSN_COST);
11663   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11664 
11665   ins_encode %{
11666     __ eonw(as_Register($dst$$reg),
11667               as_Register($src1$$reg),
11668               as_Register($src2$$reg),
11669               Assembler::ASR,
11670               $src3$$constant & 0x1f);
11671   %}
11672 
11673   ins_pipe(ialu_reg_reg_shift);
11674 %}
11675 
11676 // This pattern is automatically generated from aarch64_ad.m4
11677 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11678 // val ^ (-1 ^ (val >> shift)) ==> eon
11679 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11680                          iRegL src1, iRegL src2,
11681                          immI src3, immL_M1 src4) %{
11682   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11683   ins_cost(1.9 * INSN_COST);
11684   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11685 
11686   ins_encode %{
11687     __ eon(as_Register($dst$$reg),
11688               as_Register($src1$$reg),
11689               as_Register($src2$$reg),
11690               Assembler::ASR,
11691               $src3$$constant & 0x3f);
11692   %}
11693 
11694   ins_pipe(ialu_reg_reg_shift);
11695 %}
11696 
11697 // This pattern is automatically generated from aarch64_ad.m4
11698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11699 // val ^ (-1 ^ (val ror shift)) ==> eonw
11700 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11701                          iRegIorL2I src1, iRegIorL2I src2,
11702                          immI src3, immI_M1 src4) %{
11703   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11704   ins_cost(1.9 * INSN_COST);
11705   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11706 
11707   ins_encode %{
11708     __ eonw(as_Register($dst$$reg),
11709               as_Register($src1$$reg),
11710               as_Register($src2$$reg),
11711               Assembler::ROR,
11712               $src3$$constant & 0x1f);
11713   %}
11714 
11715   ins_pipe(ialu_reg_reg_shift);
11716 %}
11717 
11718 // This pattern is automatically generated from aarch64_ad.m4
11719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11720 // val ^ (-1 ^ (val ror shift)) ==> eon
11721 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11722                          iRegL src1, iRegL src2,
11723                          immI src3, immL_M1 src4) %{
11724   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11725   ins_cost(1.9 * INSN_COST);
11726   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11727 
11728   ins_encode %{
11729     __ eon(as_Register($dst$$reg),
11730               as_Register($src1$$reg),
11731               as_Register($src2$$reg),
11732               Assembler::ROR,
11733               $src3$$constant & 0x3f);
11734   %}
11735 
11736   ins_pipe(ialu_reg_reg_shift);
11737 %}
11738 
11739 // This pattern is automatically generated from aarch64_ad.m4
11740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11741 // val ^ (-1 ^ (val << shift)) ==> eonw
11742 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11743                          iRegIorL2I src1, iRegIorL2I src2,
11744                          immI src3, immI_M1 src4) %{
11745   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11746   ins_cost(1.9 * INSN_COST);
11747   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11748 
11749   ins_encode %{
11750     __ eonw(as_Register($dst$$reg),
11751               as_Register($src1$$reg),
11752               as_Register($src2$$reg),
11753               Assembler::LSL,
11754               $src3$$constant & 0x1f);
11755   %}
11756 
11757   ins_pipe(ialu_reg_reg_shift);
11758 %}
11759 
11760 // This pattern is automatically generated from aarch64_ad.m4
11761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11762 // val ^ (-1 ^ (val << shift)) ==> eon
11763 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11764                          iRegL src1, iRegL src2,
11765                          immI src3, immL_M1 src4) %{
11766   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11767   ins_cost(1.9 * INSN_COST);
11768   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11769 
11770   ins_encode %{
11771     __ eon(as_Register($dst$$reg),
11772               as_Register($src1$$reg),
11773               as_Register($src2$$reg),
11774               Assembler::LSL,
11775               $src3$$constant & 0x3f);
11776   %}
11777 
11778   ins_pipe(ialu_reg_reg_shift);
11779 %}
11780 
11781 // This pattern is automatically generated from aarch64_ad.m4
11782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11783 // val | (-1 ^ (val >>> shift)) ==> ornw
11784 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11785                          iRegIorL2I src1, iRegIorL2I src2,
11786                          immI src3, immI_M1 src4) %{
11787   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11788   ins_cost(1.9 * INSN_COST);
11789   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11790 
11791   ins_encode %{
11792     __ ornw(as_Register($dst$$reg),
11793               as_Register($src1$$reg),
11794               as_Register($src2$$reg),
11795               Assembler::LSR,
11796               $src3$$constant & 0x1f);
11797   %}
11798 
11799   ins_pipe(ialu_reg_reg_shift);
11800 %}
11801 
11802 // This pattern is automatically generated from aarch64_ad.m4
11803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11804 // val | (-1 ^ (val >>> shift)) ==> orn
11805 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11806                          iRegL src1, iRegL src2,
11807                          immI src3, immL_M1 src4) %{
11808   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11809   ins_cost(1.9 * INSN_COST);
11810   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11811 
11812   ins_encode %{
11813     __ orn(as_Register($dst$$reg),
11814               as_Register($src1$$reg),
11815               as_Register($src2$$reg),
11816               Assembler::LSR,
11817               $src3$$constant & 0x3f);
11818   %}
11819 
11820   ins_pipe(ialu_reg_reg_shift);
11821 %}
11822 
11823 // This pattern is automatically generated from aarch64_ad.m4
11824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11825 // val | (-1 ^ (val >> shift)) ==> ornw
11826 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11827                          iRegIorL2I src1, iRegIorL2I src2,
11828                          immI src3, immI_M1 src4) %{
11829   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11830   ins_cost(1.9 * INSN_COST);
11831   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11832 
11833   ins_encode %{
11834     __ ornw(as_Register($dst$$reg),
11835               as_Register($src1$$reg),
11836               as_Register($src2$$reg),
11837               Assembler::ASR,
11838               $src3$$constant & 0x1f);
11839   %}
11840 
11841   ins_pipe(ialu_reg_reg_shift);
11842 %}
11843 
11844 // This pattern is automatically generated from aarch64_ad.m4
11845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11846 // val | (-1 ^ (val >> shift)) ==> orn
11847 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11848                          iRegL src1, iRegL src2,
11849                          immI src3, immL_M1 src4) %{
11850   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11851   ins_cost(1.9 * INSN_COST);
11852   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11853 
11854   ins_encode %{
11855     __ orn(as_Register($dst$$reg),
11856               as_Register($src1$$reg),
11857               as_Register($src2$$reg),
11858               Assembler::ASR,
11859               $src3$$constant & 0x3f);
11860   %}
11861 
11862   ins_pipe(ialu_reg_reg_shift);
11863 %}
11864 
11865 // This pattern is automatically generated from aarch64_ad.m4
11866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11867 // val | (-1 ^ (val ror shift)) ==> ornw
11868 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11869                          iRegIorL2I src1, iRegIorL2I src2,
11870                          immI src3, immI_M1 src4) %{
11871   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11872   ins_cost(1.9 * INSN_COST);
11873   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11874 
11875   ins_encode %{
11876     __ ornw(as_Register($dst$$reg),
11877               as_Register($src1$$reg),
11878               as_Register($src2$$reg),
11879               Assembler::ROR,
11880               $src3$$constant & 0x1f);
11881   %}
11882 
11883   ins_pipe(ialu_reg_reg_shift);
11884 %}
11885 
11886 // This pattern is automatically generated from aarch64_ad.m4
11887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11888 // val | (-1 ^ (val ror shift)) ==> orn
11889 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11890                          iRegL src1, iRegL src2,
11891                          immI src3, immL_M1 src4) %{
11892   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11893   ins_cost(1.9 * INSN_COST);
11894   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11895 
11896   ins_encode %{
11897     __ orn(as_Register($dst$$reg),
11898               as_Register($src1$$reg),
11899               as_Register($src2$$reg),
11900               Assembler::ROR,
11901               $src3$$constant & 0x3f);
11902   %}
11903 
11904   ins_pipe(ialu_reg_reg_shift);
11905 %}
11906 
11907 // This pattern is automatically generated from aarch64_ad.m4
11908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11909 // val | (-1 ^ (val << shift)) ==> ornw
11910 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11911                          iRegIorL2I src1, iRegIorL2I src2,
11912                          immI src3, immI_M1 src4) %{
11913   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11914   ins_cost(1.9 * INSN_COST);
11915   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11916 
11917   ins_encode %{
11918     __ ornw(as_Register($dst$$reg),
11919               as_Register($src1$$reg),
11920               as_Register($src2$$reg),
11921               Assembler::LSL,
11922               $src3$$constant & 0x1f);
11923   %}
11924 
11925   ins_pipe(ialu_reg_reg_shift);
11926 %}
11927 
11928 // This pattern is automatically generated from aarch64_ad.m4
11929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11930 // val | (-1 ^ (val << shift)) ==> orn
11931 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11932                          iRegL src1, iRegL src2,
11933                          immI src3, immL_M1 src4) %{
11934   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11935   ins_cost(1.9 * INSN_COST);
11936   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11937 
11938   ins_encode %{
11939     __ orn(as_Register($dst$$reg),
11940               as_Register($src1$$reg),
11941               as_Register($src2$$reg),
11942               Assembler::LSL,
11943               $src3$$constant & 0x3f);
11944   %}
11945 
11946   ins_pipe(ialu_reg_reg_shift);
11947 %}
11948 
11949 // This pattern is automatically generated from aarch64_ad.m4
11950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11951 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11952                          iRegIorL2I src1, iRegIorL2I src2,
11953                          immI src3) %{
11954   match(Set dst (AndI src1 (URShiftI src2 src3)));
11955 
11956   ins_cost(1.9 * INSN_COST);
11957   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11958 
11959   ins_encode %{
11960     __ andw(as_Register($dst$$reg),
11961               as_Register($src1$$reg),
11962               as_Register($src2$$reg),
11963               Assembler::LSR,
11964               $src3$$constant & 0x1f);
11965   %}
11966 
11967   ins_pipe(ialu_reg_reg_shift);
11968 %}
11969 
11970 // This pattern is automatically generated from aarch64_ad.m4
11971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11972 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11973                          iRegL src1, iRegL src2,
11974                          immI src3) %{
11975   match(Set dst (AndL src1 (URShiftL src2 src3)));
11976 
11977   ins_cost(1.9 * INSN_COST);
11978   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11979 
11980   ins_encode %{
11981     __ andr(as_Register($dst$$reg),
11982               as_Register($src1$$reg),
11983               as_Register($src2$$reg),
11984               Assembler::LSR,
11985               $src3$$constant & 0x3f);
11986   %}
11987 
11988   ins_pipe(ialu_reg_reg_shift);
11989 %}
11990 
11991 // This pattern is automatically generated from aarch64_ad.m4
11992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11993 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11994                          iRegIorL2I src1, iRegIorL2I src2,
11995                          immI src3) %{
11996   match(Set dst (AndI src1 (RShiftI src2 src3)));
11997 
11998   ins_cost(1.9 * INSN_COST);
11999   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12000 
12001   ins_encode %{
12002     __ andw(as_Register($dst$$reg),
12003               as_Register($src1$$reg),
12004               as_Register($src2$$reg),
12005               Assembler::ASR,
12006               $src3$$constant & 0x1f);
12007   %}
12008 
12009   ins_pipe(ialu_reg_reg_shift);
12010 %}
12011 
12012 // This pattern is automatically generated from aarch64_ad.m4
12013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12014 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12015                          iRegL src1, iRegL src2,
12016                          immI src3) %{
12017   match(Set dst (AndL src1 (RShiftL src2 src3)));
12018 
12019   ins_cost(1.9 * INSN_COST);
12020   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12021 
12022   ins_encode %{
12023     __ andr(as_Register($dst$$reg),
12024               as_Register($src1$$reg),
12025               as_Register($src2$$reg),
12026               Assembler::ASR,
12027               $src3$$constant & 0x3f);
12028   %}
12029 
12030   ins_pipe(ialu_reg_reg_shift);
12031 %}
12032 
12033 // This pattern is automatically generated from aarch64_ad.m4
12034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12035 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12036                          iRegIorL2I src1, iRegIorL2I src2,
12037                          immI src3) %{
12038   match(Set dst (AndI src1 (LShiftI src2 src3)));
12039 
12040   ins_cost(1.9 * INSN_COST);
12041   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12042 
12043   ins_encode %{
12044     __ andw(as_Register($dst$$reg),
12045               as_Register($src1$$reg),
12046               as_Register($src2$$reg),
12047               Assembler::LSL,
12048               $src3$$constant & 0x1f);
12049   %}
12050 
12051   ins_pipe(ialu_reg_reg_shift);
12052 %}
12053 
12054 // This pattern is automatically generated from aarch64_ad.m4
12055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12056 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12057                          iRegL src1, iRegL src2,
12058                          immI src3) %{
12059   match(Set dst (AndL src1 (LShiftL src2 src3)));
12060 
12061   ins_cost(1.9 * INSN_COST);
12062   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12063 
12064   ins_encode %{
12065     __ andr(as_Register($dst$$reg),
12066               as_Register($src1$$reg),
12067               as_Register($src2$$reg),
12068               Assembler::LSL,
12069               $src3$$constant & 0x3f);
12070   %}
12071 
12072   ins_pipe(ialu_reg_reg_shift);
12073 %}
12074 
12075 // This pattern is automatically generated from aarch64_ad.m4
12076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12077 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12078                          iRegIorL2I src1, iRegIorL2I src2,
12079                          immI src3) %{
12080   match(Set dst (AndI src1 (RotateRight src2 src3)));
12081 
12082   ins_cost(1.9 * INSN_COST);
12083   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12084 
12085   ins_encode %{
12086     __ andw(as_Register($dst$$reg),
12087               as_Register($src1$$reg),
12088               as_Register($src2$$reg),
12089               Assembler::ROR,
12090               $src3$$constant & 0x1f);
12091   %}
12092 
12093   ins_pipe(ialu_reg_reg_shift);
12094 %}
12095 
12096 // This pattern is automatically generated from aarch64_ad.m4
12097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12098 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12099                          iRegL src1, iRegL src2,
12100                          immI src3) %{
12101   match(Set dst (AndL src1 (RotateRight src2 src3)));
12102 
12103   ins_cost(1.9 * INSN_COST);
12104   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12105 
12106   ins_encode %{
12107     __ andr(as_Register($dst$$reg),
12108               as_Register($src1$$reg),
12109               as_Register($src2$$reg),
12110               Assembler::ROR,
12111               $src3$$constant & 0x3f);
12112   %}
12113 
12114   ins_pipe(ialu_reg_reg_shift);
12115 %}
12116 
12117 // This pattern is automatically generated from aarch64_ad.m4
12118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12119 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12120                          iRegIorL2I src1, iRegIorL2I src2,
12121                          immI src3) %{
12122   match(Set dst (XorI src1 (URShiftI src2 src3)));
12123 
12124   ins_cost(1.9 * INSN_COST);
12125   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12126 
12127   ins_encode %{
12128     __ eorw(as_Register($dst$$reg),
12129               as_Register($src1$$reg),
12130               as_Register($src2$$reg),
12131               Assembler::LSR,
12132               $src3$$constant & 0x1f);
12133   %}
12134 
12135   ins_pipe(ialu_reg_reg_shift);
12136 %}
12137 
12138 // This pattern is automatically generated from aarch64_ad.m4
12139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12140 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12141                          iRegL src1, iRegL src2,
12142                          immI src3) %{
12143   match(Set dst (XorL src1 (URShiftL src2 src3)));
12144 
12145   ins_cost(1.9 * INSN_COST);
12146   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12147 
12148   ins_encode %{
12149     __ eor(as_Register($dst$$reg),
12150               as_Register($src1$$reg),
12151               as_Register($src2$$reg),
12152               Assembler::LSR,
12153               $src3$$constant & 0x3f);
12154   %}
12155 
12156   ins_pipe(ialu_reg_reg_shift);
12157 %}
12158 
12159 // This pattern is automatically generated from aarch64_ad.m4
12160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12161 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12162                          iRegIorL2I src1, iRegIorL2I src2,
12163                          immI src3) %{
12164   match(Set dst (XorI src1 (RShiftI src2 src3)));
12165 
12166   ins_cost(1.9 * INSN_COST);
12167   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12168 
12169   ins_encode %{
12170     __ eorw(as_Register($dst$$reg),
12171               as_Register($src1$$reg),
12172               as_Register($src2$$reg),
12173               Assembler::ASR,
12174               $src3$$constant & 0x1f);
12175   %}
12176 
12177   ins_pipe(ialu_reg_reg_shift);
12178 %}
12179 
12180 // This pattern is automatically generated from aarch64_ad.m4
12181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12182 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12183                          iRegL src1, iRegL src2,
12184                          immI src3) %{
12185   match(Set dst (XorL src1 (RShiftL src2 src3)));
12186 
12187   ins_cost(1.9 * INSN_COST);
12188   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12189 
12190   ins_encode %{
12191     __ eor(as_Register($dst$$reg),
12192               as_Register($src1$$reg),
12193               as_Register($src2$$reg),
12194               Assembler::ASR,
12195               $src3$$constant & 0x3f);
12196   %}
12197 
12198   ins_pipe(ialu_reg_reg_shift);
12199 %}
12200 
12201 // This pattern is automatically generated from aarch64_ad.m4
12202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12203 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12204                          iRegIorL2I src1, iRegIorL2I src2,
12205                          immI src3) %{
12206   match(Set dst (XorI src1 (LShiftI src2 src3)));
12207 
12208   ins_cost(1.9 * INSN_COST);
12209   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12210 
12211   ins_encode %{
12212     __ eorw(as_Register($dst$$reg),
12213               as_Register($src1$$reg),
12214               as_Register($src2$$reg),
12215               Assembler::LSL,
12216               $src3$$constant & 0x1f);
12217   %}
12218 
12219   ins_pipe(ialu_reg_reg_shift);
12220 %}
12221 
12222 // This pattern is automatically generated from aarch64_ad.m4
12223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12224 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12225                          iRegL src1, iRegL src2,
12226                          immI src3) %{
12227   match(Set dst (XorL src1 (LShiftL src2 src3)));
12228 
12229   ins_cost(1.9 * INSN_COST);
12230   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12231 
12232   ins_encode %{
12233     __ eor(as_Register($dst$$reg),
12234               as_Register($src1$$reg),
12235               as_Register($src2$$reg),
12236               Assembler::LSL,
12237               $src3$$constant & 0x3f);
12238   %}
12239 
12240   ins_pipe(ialu_reg_reg_shift);
12241 %}
12242 
12243 // This pattern is automatically generated from aarch64_ad.m4
12244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12245 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12246                          iRegIorL2I src1, iRegIorL2I src2,
12247                          immI src3) %{
12248   match(Set dst (XorI src1 (RotateRight src2 src3)));
12249 
12250   ins_cost(1.9 * INSN_COST);
12251   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12252 
12253   ins_encode %{
12254     __ eorw(as_Register($dst$$reg),
12255               as_Register($src1$$reg),
12256               as_Register($src2$$reg),
12257               Assembler::ROR,
12258               $src3$$constant & 0x1f);
12259   %}
12260 
12261   ins_pipe(ialu_reg_reg_shift);
12262 %}
12263 
12264 // This pattern is automatically generated from aarch64_ad.m4
12265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12266 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12267                          iRegL src1, iRegL src2,
12268                          immI src3) %{
12269   match(Set dst (XorL src1 (RotateRight src2 src3)));
12270 
12271   ins_cost(1.9 * INSN_COST);
12272   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12273 
12274   ins_encode %{
12275     __ eor(as_Register($dst$$reg),
12276               as_Register($src1$$reg),
12277               as_Register($src2$$reg),
12278               Assembler::ROR,
12279               $src3$$constant & 0x3f);
12280   %}
12281 
12282   ins_pipe(ialu_reg_reg_shift);
12283 %}
12284 
12285 // This pattern is automatically generated from aarch64_ad.m4
12286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12287 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12288                          iRegIorL2I src1, iRegIorL2I src2,
12289                          immI src3) %{
12290   match(Set dst (OrI src1 (URShiftI src2 src3)));
12291 
12292   ins_cost(1.9 * INSN_COST);
12293   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12294 
12295   ins_encode %{
12296     __ orrw(as_Register($dst$$reg),
12297               as_Register($src1$$reg),
12298               as_Register($src2$$reg),
12299               Assembler::LSR,
12300               $src3$$constant & 0x1f);
12301   %}
12302 
12303   ins_pipe(ialu_reg_reg_shift);
12304 %}
12305 
12306 // This pattern is automatically generated from aarch64_ad.m4
12307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12308 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12309                          iRegL src1, iRegL src2,
12310                          immI src3) %{
12311   match(Set dst (OrL src1 (URShiftL src2 src3)));
12312 
12313   ins_cost(1.9 * INSN_COST);
12314   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12315 
12316   ins_encode %{
12317     __ orr(as_Register($dst$$reg),
12318               as_Register($src1$$reg),
12319               as_Register($src2$$reg),
12320               Assembler::LSR,
12321               $src3$$constant & 0x3f);
12322   %}
12323 
12324   ins_pipe(ialu_reg_reg_shift);
12325 %}
12326 
12327 // This pattern is automatically generated from aarch64_ad.m4
12328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12329 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12330                          iRegIorL2I src1, iRegIorL2I src2,
12331                          immI src3) %{
12332   match(Set dst (OrI src1 (RShiftI src2 src3)));
12333 
12334   ins_cost(1.9 * INSN_COST);
12335   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12336 
12337   ins_encode %{
12338     __ orrw(as_Register($dst$$reg),
12339               as_Register($src1$$reg),
12340               as_Register($src2$$reg),
12341               Assembler::ASR,
12342               $src3$$constant & 0x1f);
12343   %}
12344 
12345   ins_pipe(ialu_reg_reg_shift);
12346 %}
12347 
12348 // This pattern is automatically generated from aarch64_ad.m4
12349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12350 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12351                          iRegL src1, iRegL src2,
12352                          immI src3) %{
12353   match(Set dst (OrL src1 (RShiftL src2 src3)));
12354 
12355   ins_cost(1.9 * INSN_COST);
12356   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12357 
12358   ins_encode %{
12359     __ orr(as_Register($dst$$reg),
12360               as_Register($src1$$reg),
12361               as_Register($src2$$reg),
12362               Assembler::ASR,
12363               $src3$$constant & 0x3f);
12364   %}
12365 
12366   ins_pipe(ialu_reg_reg_shift);
12367 %}
12368 
12369 // This pattern is automatically generated from aarch64_ad.m4
12370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12371 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12372                          iRegIorL2I src1, iRegIorL2I src2,
12373                          immI src3) %{
12374   match(Set dst (OrI src1 (LShiftI src2 src3)));
12375 
12376   ins_cost(1.9 * INSN_COST);
12377   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12378 
12379   ins_encode %{
12380     __ orrw(as_Register($dst$$reg),
12381               as_Register($src1$$reg),
12382               as_Register($src2$$reg),
12383               Assembler::LSL,
12384               $src3$$constant & 0x1f);
12385   %}
12386 
12387   ins_pipe(ialu_reg_reg_shift);
12388 %}
12389 
12390 // This pattern is automatically generated from aarch64_ad.m4
12391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12392 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12393                          iRegL src1, iRegL src2,
12394                          immI src3) %{
12395   match(Set dst (OrL src1 (LShiftL src2 src3)));
12396 
12397   ins_cost(1.9 * INSN_COST);
12398   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12399 
12400   ins_encode %{
12401     __ orr(as_Register($dst$$reg),
12402               as_Register($src1$$reg),
12403               as_Register($src2$$reg),
12404               Assembler::LSL,
12405               $src3$$constant & 0x3f);
12406   %}
12407 
12408   ins_pipe(ialu_reg_reg_shift);
12409 %}
12410 
12411 // This pattern is automatically generated from aarch64_ad.m4
12412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12413 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12414                          iRegIorL2I src1, iRegIorL2I src2,
12415                          immI src3) %{
12416   match(Set dst (OrI src1 (RotateRight src2 src3)));
12417 
12418   ins_cost(1.9 * INSN_COST);
12419   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12420 
12421   ins_encode %{
12422     __ orrw(as_Register($dst$$reg),
12423               as_Register($src1$$reg),
12424               as_Register($src2$$reg),
12425               Assembler::ROR,
12426               $src3$$constant & 0x1f);
12427   %}
12428 
12429   ins_pipe(ialu_reg_reg_shift);
12430 %}
12431 
12432 // This pattern is automatically generated from aarch64_ad.m4
12433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12434 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12435                          iRegL src1, iRegL src2,
12436                          immI src3) %{
12437   match(Set dst (OrL src1 (RotateRight src2 src3)));
12438 
12439   ins_cost(1.9 * INSN_COST);
12440   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12441 
12442   ins_encode %{
12443     __ orr(as_Register($dst$$reg),
12444               as_Register($src1$$reg),
12445               as_Register($src2$$reg),
12446               Assembler::ROR,
12447               $src3$$constant & 0x3f);
12448   %}
12449 
12450   ins_pipe(ialu_reg_reg_shift);
12451 %}
12452 
12453 // This pattern is automatically generated from aarch64_ad.m4
12454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12455 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12456                          iRegIorL2I src1, iRegIorL2I src2,
12457                          immI src3) %{
12458   match(Set dst (AddI src1 (URShiftI src2 src3)));
12459 
12460   ins_cost(1.9 * INSN_COST);
12461   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12462 
12463   ins_encode %{
12464     __ addw(as_Register($dst$$reg),
12465               as_Register($src1$$reg),
12466               as_Register($src2$$reg),
12467               Assembler::LSR,
12468               $src3$$constant & 0x1f);
12469   %}
12470 
12471   ins_pipe(ialu_reg_reg_shift);
12472 %}
12473 
12474 // This pattern is automatically generated from aarch64_ad.m4
12475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12476 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12477                          iRegL src1, iRegL src2,
12478                          immI src3) %{
12479   match(Set dst (AddL src1 (URShiftL src2 src3)));
12480 
12481   ins_cost(1.9 * INSN_COST);
12482   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12483 
12484   ins_encode %{
12485     __ add(as_Register($dst$$reg),
12486               as_Register($src1$$reg),
12487               as_Register($src2$$reg),
12488               Assembler::LSR,
12489               $src3$$constant & 0x3f);
12490   %}
12491 
12492   ins_pipe(ialu_reg_reg_shift);
12493 %}
12494 
12495 // This pattern is automatically generated from aarch64_ad.m4
12496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12497 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12498                          iRegIorL2I src1, iRegIorL2I src2,
12499                          immI src3) %{
12500   match(Set dst (AddI src1 (RShiftI src2 src3)));
12501 
12502   ins_cost(1.9 * INSN_COST);
12503   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12504 
12505   ins_encode %{
12506     __ addw(as_Register($dst$$reg),
12507               as_Register($src1$$reg),
12508               as_Register($src2$$reg),
12509               Assembler::ASR,
12510               $src3$$constant & 0x1f);
12511   %}
12512 
12513   ins_pipe(ialu_reg_reg_shift);
12514 %}
12515 
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12519                          iRegL src1, iRegL src2,
12520                          immI src3) %{
12521   match(Set dst (AddL src1 (RShiftL src2 src3)));
12522 
12523   ins_cost(1.9 * INSN_COST);
12524   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12525 
12526   ins_encode %{
12527     __ add(as_Register($dst$$reg),
12528               as_Register($src1$$reg),
12529               as_Register($src2$$reg),
12530               Assembler::ASR,
12531               $src3$$constant & 0x3f);
12532   %}
12533 
12534   ins_pipe(ialu_reg_reg_shift);
12535 %}
12536 
12537 // This pattern is automatically generated from aarch64_ad.m4
12538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12539 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12540                          iRegIorL2I src1, iRegIorL2I src2,
12541                          immI src3) %{
12542   match(Set dst (AddI src1 (LShiftI src2 src3)));
12543 
12544   ins_cost(1.9 * INSN_COST);
12545   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12546 
12547   ins_encode %{
12548     __ addw(as_Register($dst$$reg),
12549               as_Register($src1$$reg),
12550               as_Register($src2$$reg),
12551               Assembler::LSL,
12552               $src3$$constant & 0x1f);
12553   %}
12554 
12555   ins_pipe(ialu_reg_reg_shift);
12556 %}
12557 
12558 // This pattern is automatically generated from aarch64_ad.m4
12559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12560 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12561                          iRegL src1, iRegL src2,
12562                          immI src3) %{
12563   match(Set dst (AddL src1 (LShiftL src2 src3)));
12564 
12565   ins_cost(1.9 * INSN_COST);
12566   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12567 
12568   ins_encode %{
12569     __ add(as_Register($dst$$reg),
12570               as_Register($src1$$reg),
12571               as_Register($src2$$reg),
12572               Assembler::LSL,
12573               $src3$$constant & 0x3f);
12574   %}
12575 
12576   ins_pipe(ialu_reg_reg_shift);
12577 %}
12578 
12579 // This pattern is automatically generated from aarch64_ad.m4
12580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12581 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12582                          iRegIorL2I src1, iRegIorL2I src2,
12583                          immI src3) %{
12584   match(Set dst (SubI src1 (URShiftI src2 src3)));
12585 
12586   ins_cost(1.9 * INSN_COST);
12587   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12588 
12589   ins_encode %{
12590     __ subw(as_Register($dst$$reg),
12591               as_Register($src1$$reg),
12592               as_Register($src2$$reg),
12593               Assembler::LSR,
12594               $src3$$constant & 0x1f);
12595   %}
12596 
12597   ins_pipe(ialu_reg_reg_shift);
12598 %}
12599 
12600 // This pattern is automatically generated from aarch64_ad.m4
12601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12602 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12603                          iRegL src1, iRegL src2,
12604                          immI src3) %{
12605   match(Set dst (SubL src1 (URShiftL src2 src3)));
12606 
12607   ins_cost(1.9 * INSN_COST);
12608   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12609 
12610   ins_encode %{
12611     __ sub(as_Register($dst$$reg),
12612               as_Register($src1$$reg),
12613               as_Register($src2$$reg),
12614               Assembler::LSR,
12615               $src3$$constant & 0x3f);
12616   %}
12617 
12618   ins_pipe(ialu_reg_reg_shift);
12619 %}
12620 
12621 // This pattern is automatically generated from aarch64_ad.m4
12622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12623 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12624                          iRegIorL2I src1, iRegIorL2I src2,
12625                          immI src3) %{
12626   match(Set dst (SubI src1 (RShiftI src2 src3)));
12627 
12628   ins_cost(1.9 * INSN_COST);
12629   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12630 
12631   ins_encode %{
12632     __ subw(as_Register($dst$$reg),
12633               as_Register($src1$$reg),
12634               as_Register($src2$$reg),
12635               Assembler::ASR,
12636               $src3$$constant & 0x1f);
12637   %}
12638 
12639   ins_pipe(ialu_reg_reg_shift);
12640 %}
12641 
12642 // This pattern is automatically generated from aarch64_ad.m4
12643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12644 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12645                          iRegL src1, iRegL src2,
12646                          immI src3) %{
12647   match(Set dst (SubL src1 (RShiftL src2 src3)));
12648 
12649   ins_cost(1.9 * INSN_COST);
12650   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12651 
12652   ins_encode %{
12653     __ sub(as_Register($dst$$reg),
12654               as_Register($src1$$reg),
12655               as_Register($src2$$reg),
12656               Assembler::ASR,
12657               $src3$$constant & 0x3f);
12658   %}
12659 
12660   ins_pipe(ialu_reg_reg_shift);
12661 %}
12662 
12663 // This pattern is automatically generated from aarch64_ad.m4
12664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12665 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12666                          iRegIorL2I src1, iRegIorL2I src2,
12667                          immI src3) %{
12668   match(Set dst (SubI src1 (LShiftI src2 src3)));
12669 
12670   ins_cost(1.9 * INSN_COST);
12671   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12672 
12673   ins_encode %{
12674     __ subw(as_Register($dst$$reg),
12675               as_Register($src1$$reg),
12676               as_Register($src2$$reg),
12677               Assembler::LSL,
12678               $src3$$constant & 0x1f);
12679   %}
12680 
12681   ins_pipe(ialu_reg_reg_shift);
12682 %}
12683 
12684 // This pattern is automatically generated from aarch64_ad.m4
12685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12686 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12687                          iRegL src1, iRegL src2,
12688                          immI src3) %{
12689   match(Set dst (SubL src1 (LShiftL src2 src3)));
12690 
12691   ins_cost(1.9 * INSN_COST);
12692   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12693 
12694   ins_encode %{
12695     __ sub(as_Register($dst$$reg),
12696               as_Register($src1$$reg),
12697               as_Register($src2$$reg),
12698               Assembler::LSL,
12699               $src3$$constant & 0x3f);
12700   %}
12701 
12702   ins_pipe(ialu_reg_reg_shift);
12703 %}
12704 
12705 // This pattern is automatically generated from aarch64_ad.m4
12706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12707 
12708 // Shift Left followed by Shift Right.
12709 // This idiom is used by the compiler for the i2b bytecode etc.
12710 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12711 %{
12712   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12713   ins_cost(INSN_COST * 2);
12714   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12715   ins_encode %{
12716     int lshift = $lshift_count$$constant & 63;
12717     int rshift = $rshift_count$$constant & 63;
12718     int s = 63 - lshift;
12719     int r = (rshift - lshift) & 63;
12720     __ sbfm(as_Register($dst$$reg),
12721             as_Register($src$$reg),
12722             r, s);
12723   %}
12724 
12725   ins_pipe(ialu_reg_shift);
12726 %}
12727 
12728 // This pattern is automatically generated from aarch64_ad.m4
12729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12730 
12731 // Shift Left followed by Shift Right.
12732 // This idiom is used by the compiler for the i2b bytecode etc.
12733 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12734 %{
12735   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12736   ins_cost(INSN_COST * 2);
12737   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12738   ins_encode %{
12739     int lshift = $lshift_count$$constant & 31;
12740     int rshift = $rshift_count$$constant & 31;
12741     int s = 31 - lshift;
12742     int r = (rshift - lshift) & 31;
12743     __ sbfmw(as_Register($dst$$reg),
12744             as_Register($src$$reg),
12745             r, s);
12746   %}
12747 
12748   ins_pipe(ialu_reg_shift);
12749 %}
12750 
12751 // This pattern is automatically generated from aarch64_ad.m4
12752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12753 
12754 // Shift Left followed by Shift Right.
12755 // This idiom is used by the compiler for the i2b bytecode etc.
12756 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12757 %{
12758   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12759   ins_cost(INSN_COST * 2);
12760   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12761   ins_encode %{
12762     int lshift = $lshift_count$$constant & 63;
12763     int rshift = $rshift_count$$constant & 63;
12764     int s = 63 - lshift;
12765     int r = (rshift - lshift) & 63;
12766     __ ubfm(as_Register($dst$$reg),
12767             as_Register($src$$reg),
12768             r, s);
12769   %}
12770 
12771   ins_pipe(ialu_reg_shift);
12772 %}
12773 
12774 // This pattern is automatically generated from aarch64_ad.m4
12775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12776 
12777 // Shift Left followed by Shift Right.
12778 // This idiom is used by the compiler for the i2b bytecode etc.
12779 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12780 %{
12781   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12782   ins_cost(INSN_COST * 2);
12783   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12784   ins_encode %{
12785     int lshift = $lshift_count$$constant & 31;
12786     int rshift = $rshift_count$$constant & 31;
12787     int s = 31 - lshift;
12788     int r = (rshift - lshift) & 31;
12789     __ ubfmw(as_Register($dst$$reg),
12790             as_Register($src$$reg),
12791             r, s);
12792   %}
12793 
12794   ins_pipe(ialu_reg_shift);
12795 %}
12796 
12797 // Bitfield extract with shift & mask
12798 
12799 // This pattern is automatically generated from aarch64_ad.m4
12800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12801 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12802 %{
12803   match(Set dst (AndI (URShiftI src rshift) mask));
12804   // Make sure we are not going to exceed what ubfxw can do.
12805   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12806 
12807   ins_cost(INSN_COST);
12808   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12809   ins_encode %{
12810     int rshift = $rshift$$constant & 31;
12811     intptr_t mask = $mask$$constant;
12812     int width = exact_log2(mask+1);
12813     __ ubfxw(as_Register($dst$$reg),
12814             as_Register($src$$reg), rshift, width);
12815   %}
12816   ins_pipe(ialu_reg_shift);
12817 %}
12818 
12819 // This pattern is automatically generated from aarch64_ad.m4
12820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12821 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12822 %{
12823   match(Set dst (AndL (URShiftL src rshift) mask));
12824   // Make sure we are not going to exceed what ubfx can do.
12825   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12826 
12827   ins_cost(INSN_COST);
12828   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12829   ins_encode %{
12830     int rshift = $rshift$$constant & 63;
12831     intptr_t mask = $mask$$constant;
12832     int width = exact_log2_long(mask+1);
12833     __ ubfx(as_Register($dst$$reg),
12834             as_Register($src$$reg), rshift, width);
12835   %}
12836   ins_pipe(ialu_reg_shift);
12837 %}
12838 
12839 
12840 // This pattern is automatically generated from aarch64_ad.m4
12841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12842 
12843 // We can use ubfx when extending an And with a mask when we know mask
12844 // is positive.  We know that because immI_bitmask guarantees it.
12845 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12846 %{
12847   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12848   // Make sure we are not going to exceed what ubfxw can do.
12849   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12850 
12851   ins_cost(INSN_COST * 2);
12852   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12853   ins_encode %{
12854     int rshift = $rshift$$constant & 31;
12855     intptr_t mask = $mask$$constant;
12856     int width = exact_log2(mask+1);
12857     __ ubfx(as_Register($dst$$reg),
12858             as_Register($src$$reg), rshift, width);
12859   %}
12860   ins_pipe(ialu_reg_shift);
12861 %}
12862 
12863 
12864 // This pattern is automatically generated from aarch64_ad.m4
12865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12866 
12867 // We can use ubfiz when masking by a positive number and then left shifting the result.
12868 // We know that the mask is positive because immI_bitmask guarantees it.
12869 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12870 %{
12871   match(Set dst (LShiftI (AndI src mask) lshift));
12872   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12873 
12874   ins_cost(INSN_COST);
12875   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12876   ins_encode %{
12877     int lshift = $lshift$$constant & 31;
12878     intptr_t mask = $mask$$constant;
12879     int width = exact_log2(mask+1);
12880     __ ubfizw(as_Register($dst$$reg),
12881           as_Register($src$$reg), lshift, width);
12882   %}
12883   ins_pipe(ialu_reg_shift);
12884 %}
12885 
12886 // This pattern is automatically generated from aarch64_ad.m4
12887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12888 
12889 // We can use ubfiz when masking by a positive number and then left shifting the result.
12890 // We know that the mask is positive because immL_bitmask guarantees it.
12891 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12892 %{
12893   match(Set dst (LShiftL (AndL src mask) lshift));
12894   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12895 
12896   ins_cost(INSN_COST);
12897   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12898   ins_encode %{
12899     int lshift = $lshift$$constant & 63;
12900     intptr_t mask = $mask$$constant;
12901     int width = exact_log2_long(mask+1);
12902     __ ubfiz(as_Register($dst$$reg),
12903           as_Register($src$$reg), lshift, width);
12904   %}
12905   ins_pipe(ialu_reg_shift);
12906 %}
12907 
12908 // This pattern is automatically generated from aarch64_ad.m4
12909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12910 
12911 // We can use ubfiz when masking by a positive number and then left shifting the result.
12912 // We know that the mask is positive because immI_bitmask guarantees it.
12913 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12914 %{
12915   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12916   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12917 
12918   ins_cost(INSN_COST);
12919   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12920   ins_encode %{
12921     int lshift = $lshift$$constant & 31;
12922     intptr_t mask = $mask$$constant;
12923     int width = exact_log2(mask+1);
12924     __ ubfizw(as_Register($dst$$reg),
12925           as_Register($src$$reg), lshift, width);
12926   %}
12927   ins_pipe(ialu_reg_shift);
12928 %}
12929 
12930 // This pattern is automatically generated from aarch64_ad.m4
12931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12932 
12933 // We can use ubfiz when masking by a positive number and then left shifting the result.
12934 // We know that the mask is positive because immL_bitmask guarantees it.
12935 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12936 %{
12937   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12938   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12939 
12940   ins_cost(INSN_COST);
12941   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12942   ins_encode %{
12943     int lshift = $lshift$$constant & 63;
12944     intptr_t mask = $mask$$constant;
12945     int width = exact_log2_long(mask+1);
12946     __ ubfiz(as_Register($dst$$reg),
12947           as_Register($src$$reg), lshift, width);
12948   %}
12949   ins_pipe(ialu_reg_shift);
12950 %}
12951 
12952 
12953 // This pattern is automatically generated from aarch64_ad.m4
12954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12955 
12956 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12957 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12958 %{
12959   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12960   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12961 
12962   ins_cost(INSN_COST);
12963   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12964   ins_encode %{
12965     int lshift = $lshift$$constant & 63;
12966     intptr_t mask = $mask$$constant;
12967     int width = exact_log2(mask+1);
12968     __ ubfiz(as_Register($dst$$reg),
12969              as_Register($src$$reg), lshift, width);
12970   %}
12971   ins_pipe(ialu_reg_shift);
12972 %}
12973 
12974 // This pattern is automatically generated from aarch64_ad.m4
12975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12976 
12977 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12978 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12979 %{
12980   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12981   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12982 
12983   ins_cost(INSN_COST);
12984   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12985   ins_encode %{
12986     int lshift = $lshift$$constant & 31;
12987     intptr_t mask = $mask$$constant;
12988     int width = exact_log2(mask+1);
12989     __ ubfiz(as_Register($dst$$reg),
12990              as_Register($src$$reg), lshift, width);
12991   %}
12992   ins_pipe(ialu_reg_shift);
12993 %}
12994 
12995 // This pattern is automatically generated from aarch64_ad.m4
12996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12997 
12998 // Can skip int2long conversions after AND with small bitmask
12999 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13000 %{
13001   match(Set dst (ConvI2L (AndI src msk)));
13002   ins_cost(INSN_COST);
13003   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13004   ins_encode %{
13005     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13006   %}
13007   ins_pipe(ialu_reg_shift);
13008 %}
13009 
13010 
13011 // Rotations
13012 // This pattern is automatically generated from aarch64_ad.m4
13013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13014 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13015 %{
13016   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13017   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13018 
13019   ins_cost(INSN_COST);
13020   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13021 
13022   ins_encode %{
13023     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13024             $rshift$$constant & 63);
13025   %}
13026   ins_pipe(ialu_reg_reg_extr);
13027 %}
13028 
13029 
13030 // This pattern is automatically generated from aarch64_ad.m4
13031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13032 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13033 %{
13034   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13035   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13036 
13037   ins_cost(INSN_COST);
13038   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13039 
13040   ins_encode %{
13041     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13042             $rshift$$constant & 31);
13043   %}
13044   ins_pipe(ialu_reg_reg_extr);
13045 %}
13046 
13047 
13048 // This pattern is automatically generated from aarch64_ad.m4
13049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13050 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13051 %{
13052   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13053   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13054 
13055   ins_cost(INSN_COST);
13056   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13057 
13058   ins_encode %{
13059     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13060             $rshift$$constant & 63);
13061   %}
13062   ins_pipe(ialu_reg_reg_extr);
13063 %}
13064 
13065 
13066 // This pattern is automatically generated from aarch64_ad.m4
13067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13068 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13069 %{
13070   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13071   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13072 
13073   ins_cost(INSN_COST);
13074   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13075 
13076   ins_encode %{
13077     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13078             $rshift$$constant & 31);
13079   %}
13080   ins_pipe(ialu_reg_reg_extr);
13081 %}
13082 
13083 
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13087 %{
13088   match(Set dst (RotateRight src shift));
13089 
13090   ins_cost(INSN_COST);
13091   format %{ "ror    $dst, $src, $shift" %}
13092 
13093   ins_encode %{
13094      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13095                $shift$$constant & 0x1f);
13096   %}
13097   ins_pipe(ialu_reg_reg_vshift);
13098 %}
13099 
13100 // This pattern is automatically generated from aarch64_ad.m4
13101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13102 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13103 %{
13104   match(Set dst (RotateRight src shift));
13105 
13106   ins_cost(INSN_COST);
13107   format %{ "ror    $dst, $src, $shift" %}
13108 
13109   ins_encode %{
13110      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13111                $shift$$constant & 0x3f);
13112   %}
13113   ins_pipe(ialu_reg_reg_vshift);
13114 %}
13115 
13116 // This pattern is automatically generated from aarch64_ad.m4
13117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13118 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13119 %{
13120   match(Set dst (RotateRight src shift));
13121 
13122   ins_cost(INSN_COST);
13123   format %{ "ror    $dst, $src, $shift" %}
13124 
13125   ins_encode %{
13126      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13127   %}
13128   ins_pipe(ialu_reg_reg_vshift);
13129 %}
13130 
13131 // This pattern is automatically generated from aarch64_ad.m4
13132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13133 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13134 %{
13135   match(Set dst (RotateRight src shift));
13136 
13137   ins_cost(INSN_COST);
13138   format %{ "ror    $dst, $src, $shift" %}
13139 
13140   ins_encode %{
13141      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13142   %}
13143   ins_pipe(ialu_reg_reg_vshift);
13144 %}
13145 
13146 // This pattern is automatically generated from aarch64_ad.m4
13147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13148 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13149 %{
13150   match(Set dst (RotateLeft src shift));
13151 
13152   ins_cost(INSN_COST);
13153   format %{ "rol    $dst, $src, $shift" %}
13154 
13155   ins_encode %{
13156      __ subw(rscratch1, zr, as_Register($shift$$reg));
13157      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13158   %}
13159   ins_pipe(ialu_reg_reg_vshift);
13160 %}
13161 
13162 // This pattern is automatically generated from aarch64_ad.m4
13163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13164 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13165 %{
13166   match(Set dst (RotateLeft src shift));
13167 
13168   ins_cost(INSN_COST);
13169   format %{ "rol    $dst, $src, $shift" %}
13170 
13171   ins_encode %{
13172      __ subw(rscratch1, zr, as_Register($shift$$reg));
13173      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13174   %}
13175   ins_pipe(ialu_reg_reg_vshift);
13176 %}
13177 
13178 
13179 // Add/subtract (extended)
13180 
13181 // This pattern is automatically generated from aarch64_ad.m4
13182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13183 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13184 %{
13185   match(Set dst (AddL src1 (ConvI2L src2)));
13186   ins_cost(INSN_COST);
13187   format %{ "add  $dst, $src1, $src2, sxtw" %}
13188 
13189    ins_encode %{
13190      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13191             as_Register($src2$$reg), ext::sxtw);
13192    %}
13193   ins_pipe(ialu_reg_reg);
13194 %}
13195 
13196 // This pattern is automatically generated from aarch64_ad.m4
13197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13198 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13199 %{
13200   match(Set dst (SubL src1 (ConvI2L src2)));
13201   ins_cost(INSN_COST);
13202   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13203 
13204    ins_encode %{
13205      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13206             as_Register($src2$$reg), ext::sxtw);
13207    %}
13208   ins_pipe(ialu_reg_reg);
13209 %}
13210 
13211 // This pattern is automatically generated from aarch64_ad.m4
13212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13213 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13214 %{
13215   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13216   ins_cost(INSN_COST);
13217   format %{ "add  $dst, $src1, $src2, sxth" %}
13218 
13219    ins_encode %{
13220      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13221             as_Register($src2$$reg), ext::sxth);
13222    %}
13223   ins_pipe(ialu_reg_reg);
13224 %}
13225 
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13229 %{
13230   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13231   ins_cost(INSN_COST);
13232   format %{ "add  $dst, $src1, $src2, sxtb" %}
13233 
13234    ins_encode %{
13235      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13236             as_Register($src2$$reg), ext::sxtb);
13237    %}
13238   ins_pipe(ialu_reg_reg);
13239 %}
13240 
13241 // This pattern is automatically generated from aarch64_ad.m4
13242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13243 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13244 %{
13245   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13246   ins_cost(INSN_COST);
13247   format %{ "add  $dst, $src1, $src2, uxtb" %}
13248 
13249    ins_encode %{
13250      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13251             as_Register($src2$$reg), ext::uxtb);
13252    %}
13253   ins_pipe(ialu_reg_reg);
13254 %}
13255 
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13259 %{
13260   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13261   ins_cost(INSN_COST);
13262   format %{ "add  $dst, $src1, $src2, sxth" %}
13263 
13264    ins_encode %{
13265      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13266             as_Register($src2$$reg), ext::sxth);
13267    %}
13268   ins_pipe(ialu_reg_reg);
13269 %}
13270 
13271 // This pattern is automatically generated from aarch64_ad.m4
13272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13273 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13274 %{
13275   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13276   ins_cost(INSN_COST);
13277   format %{ "add  $dst, $src1, $src2, sxtw" %}
13278 
13279    ins_encode %{
13280      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13281             as_Register($src2$$reg), ext::sxtw);
13282    %}
13283   ins_pipe(ialu_reg_reg);
13284 %}
13285 
13286 // This pattern is automatically generated from aarch64_ad.m4
13287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13288 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13289 %{
13290   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13291   ins_cost(INSN_COST);
13292   format %{ "add  $dst, $src1, $src2, sxtb" %}
13293 
13294    ins_encode %{
13295      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13296             as_Register($src2$$reg), ext::sxtb);
13297    %}
13298   ins_pipe(ialu_reg_reg);
13299 %}
13300 
13301 // This pattern is automatically generated from aarch64_ad.m4
13302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13303 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13304 %{
13305   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13306   ins_cost(INSN_COST);
13307   format %{ "add  $dst, $src1, $src2, uxtb" %}
13308 
13309    ins_encode %{
13310      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13311             as_Register($src2$$reg), ext::uxtb);
13312    %}
13313   ins_pipe(ialu_reg_reg);
13314 %}
13315 
13316 // This pattern is automatically generated from aarch64_ad.m4
13317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13318 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13319 %{
13320   match(Set dst (AddI src1 (AndI src2 mask)));
13321   ins_cost(INSN_COST);
13322   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13323 
13324    ins_encode %{
13325      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13326             as_Register($src2$$reg), ext::uxtb);
13327    %}
13328   ins_pipe(ialu_reg_reg);
13329 %}
13330 
13331 // This pattern is automatically generated from aarch64_ad.m4
13332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13333 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13334 %{
13335   match(Set dst (AddI src1 (AndI src2 mask)));
13336   ins_cost(INSN_COST);
13337   format %{ "addw  $dst, $src1, $src2, uxth" %}
13338 
13339    ins_encode %{
13340      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13341             as_Register($src2$$reg), ext::uxth);
13342    %}
13343   ins_pipe(ialu_reg_reg);
13344 %}
13345 
13346 // This pattern is automatically generated from aarch64_ad.m4
13347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13348 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13349 %{
13350   match(Set dst (AddL src1 (AndL src2 mask)));
13351   ins_cost(INSN_COST);
13352   format %{ "add  $dst, $src1, $src2, uxtb" %}
13353 
13354    ins_encode %{
13355      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13356             as_Register($src2$$reg), ext::uxtb);
13357    %}
13358   ins_pipe(ialu_reg_reg);
13359 %}
13360 
13361 // This pattern is automatically generated from aarch64_ad.m4
13362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13363 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13364 %{
13365   match(Set dst (AddL src1 (AndL src2 mask)));
13366   ins_cost(INSN_COST);
13367   format %{ "add  $dst, $src1, $src2, uxth" %}
13368 
13369    ins_encode %{
13370      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13371             as_Register($src2$$reg), ext::uxth);
13372    %}
13373   ins_pipe(ialu_reg_reg);
13374 %}
13375 
13376 // This pattern is automatically generated from aarch64_ad.m4
13377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13378 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13379 %{
13380   match(Set dst (AddL src1 (AndL src2 mask)));
13381   ins_cost(INSN_COST);
13382   format %{ "add  $dst, $src1, $src2, uxtw" %}
13383 
13384    ins_encode %{
13385      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13386             as_Register($src2$$reg), ext::uxtw);
13387    %}
13388   ins_pipe(ialu_reg_reg);
13389 %}
13390 
13391 // This pattern is automatically generated from aarch64_ad.m4
13392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13393 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13394 %{
13395   match(Set dst (SubI src1 (AndI src2 mask)));
13396   ins_cost(INSN_COST);
13397   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13398 
13399    ins_encode %{
13400      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13401             as_Register($src2$$reg), ext::uxtb);
13402    %}
13403   ins_pipe(ialu_reg_reg);
13404 %}
13405 
13406 // This pattern is automatically generated from aarch64_ad.m4
13407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13408 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13409 %{
13410   match(Set dst (SubI src1 (AndI src2 mask)));
13411   ins_cost(INSN_COST);
13412   format %{ "subw  $dst, $src1, $src2, uxth" %}
13413 
13414    ins_encode %{
13415      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13416             as_Register($src2$$reg), ext::uxth);
13417    %}
13418   ins_pipe(ialu_reg_reg);
13419 %}
13420 
13421 // This pattern is automatically generated from aarch64_ad.m4
13422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13423 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13424 %{
13425   match(Set dst (SubL src1 (AndL src2 mask)));
13426   ins_cost(INSN_COST);
13427   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13428 
13429    ins_encode %{
13430      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13431             as_Register($src2$$reg), ext::uxtb);
13432    %}
13433   ins_pipe(ialu_reg_reg);
13434 %}
13435 
13436 // This pattern is automatically generated from aarch64_ad.m4
13437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13438 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13439 %{
13440   match(Set dst (SubL src1 (AndL src2 mask)));
13441   ins_cost(INSN_COST);
13442   format %{ "sub  $dst, $src1, $src2, uxth" %}
13443 
13444    ins_encode %{
13445      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13446             as_Register($src2$$reg), ext::uxth);
13447    %}
13448   ins_pipe(ialu_reg_reg);
13449 %}
13450 
13451 // This pattern is automatically generated from aarch64_ad.m4
13452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13453 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13454 %{
13455   match(Set dst (SubL src1 (AndL src2 mask)));
13456   ins_cost(INSN_COST);
13457   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13458 
13459    ins_encode %{
13460      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13461             as_Register($src2$$reg), ext::uxtw);
13462    %}
13463   ins_pipe(ialu_reg_reg);
13464 %}
13465 
13466 
13467 // This pattern is automatically generated from aarch64_ad.m4
13468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13469 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13470 %{
13471   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13472   ins_cost(1.9 * INSN_COST);
13473   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13474 
13475    ins_encode %{
13476      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13477             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13478    %}
13479   ins_pipe(ialu_reg_reg_shift);
13480 %}
13481 
13482 // This pattern is automatically generated from aarch64_ad.m4
13483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13484 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13485 %{
13486   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13487   ins_cost(1.9 * INSN_COST);
13488   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13489 
13490    ins_encode %{
13491      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13492             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13493    %}
13494   ins_pipe(ialu_reg_reg_shift);
13495 %}
13496 
13497 // This pattern is automatically generated from aarch64_ad.m4
13498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13499 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13500 %{
13501   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13502   ins_cost(1.9 * INSN_COST);
13503   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13504 
13505    ins_encode %{
13506      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13507             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13508    %}
13509   ins_pipe(ialu_reg_reg_shift);
13510 %}
13511 
13512 // This pattern is automatically generated from aarch64_ad.m4
13513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13514 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13515 %{
13516   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13517   ins_cost(1.9 * INSN_COST);
13518   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13519 
13520    ins_encode %{
13521      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13522             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13523    %}
13524   ins_pipe(ialu_reg_reg_shift);
13525 %}
13526 
13527 // This pattern is automatically generated from aarch64_ad.m4
13528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13529 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13530 %{
13531   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13532   ins_cost(1.9 * INSN_COST);
13533   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13534 
13535    ins_encode %{
13536      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13537             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13538    %}
13539   ins_pipe(ialu_reg_reg_shift);
13540 %}
13541 
13542 // This pattern is automatically generated from aarch64_ad.m4
13543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13544 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13545 %{
13546   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13547   ins_cost(1.9 * INSN_COST);
13548   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13549 
13550    ins_encode %{
13551      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13552             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13553    %}
13554   ins_pipe(ialu_reg_reg_shift);
13555 %}
13556 
13557 // This pattern is automatically generated from aarch64_ad.m4
13558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13559 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13560 %{
13561   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13562   ins_cost(1.9 * INSN_COST);
13563   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13564 
13565    ins_encode %{
13566      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13567             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13568    %}
13569   ins_pipe(ialu_reg_reg_shift);
13570 %}
13571 
13572 // This pattern is automatically generated from aarch64_ad.m4
13573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13574 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13575 %{
13576   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13577   ins_cost(1.9 * INSN_COST);
13578   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13579 
13580    ins_encode %{
13581      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13582             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13583    %}
13584   ins_pipe(ialu_reg_reg_shift);
13585 %}
13586 
13587 // This pattern is automatically generated from aarch64_ad.m4
13588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13589 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13590 %{
13591   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13592   ins_cost(1.9 * INSN_COST);
13593   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13594 
13595    ins_encode %{
13596      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13597             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13598    %}
13599   ins_pipe(ialu_reg_reg_shift);
13600 %}
13601 
13602 // This pattern is automatically generated from aarch64_ad.m4
13603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13604 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13605 %{
13606   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13607   ins_cost(1.9 * INSN_COST);
13608   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13609 
13610    ins_encode %{
13611      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13612             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13613    %}
13614   ins_pipe(ialu_reg_reg_shift);
13615 %}
13616 
13617 // This pattern is automatically generated from aarch64_ad.m4
13618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13619 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13620 %{
13621   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13622   ins_cost(1.9 * INSN_COST);
13623   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13624 
13625    ins_encode %{
13626      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13627             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13628    %}
13629   ins_pipe(ialu_reg_reg_shift);
13630 %}
13631 
13632 // This pattern is automatically generated from aarch64_ad.m4
13633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13634 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13635 %{
13636   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13637   ins_cost(1.9 * INSN_COST);
13638   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13639 
13640    ins_encode %{
13641      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13642             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13643    %}
13644   ins_pipe(ialu_reg_reg_shift);
13645 %}
13646 
13647 // This pattern is automatically generated from aarch64_ad.m4
13648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13649 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13650 %{
13651   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13652   ins_cost(1.9 * INSN_COST);
13653   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13654 
13655    ins_encode %{
13656      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13657             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13658    %}
13659   ins_pipe(ialu_reg_reg_shift);
13660 %}
13661 
13662 // This pattern is automatically generated from aarch64_ad.m4
13663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13664 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13665 %{
13666   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13667   ins_cost(1.9 * INSN_COST);
13668   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13669 
13670    ins_encode %{
13671      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13672             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13673    %}
13674   ins_pipe(ialu_reg_reg_shift);
13675 %}
13676 
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13680 %{
13681   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13682   ins_cost(1.9 * INSN_COST);
13683   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13684 
13685    ins_encode %{
13686      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13687             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13688    %}
13689   ins_pipe(ialu_reg_reg_shift);
13690 %}
13691 
13692 // This pattern is automatically generated from aarch64_ad.m4
13693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13694 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13695 %{
13696   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13697   ins_cost(1.9 * INSN_COST);
13698   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13699 
13700    ins_encode %{
13701      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13702             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13703    %}
13704   ins_pipe(ialu_reg_reg_shift);
13705 %}
13706 
13707 // This pattern is automatically generated from aarch64_ad.m4
13708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13709 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13710 %{
13711   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13712   ins_cost(1.9 * INSN_COST);
13713   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13714 
13715    ins_encode %{
13716      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13717             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13718    %}
13719   ins_pipe(ialu_reg_reg_shift);
13720 %}
13721 
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13725 %{
13726   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13727   ins_cost(1.9 * INSN_COST);
13728   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13729 
13730    ins_encode %{
13731      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13732             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13733    %}
13734   ins_pipe(ialu_reg_reg_shift);
13735 %}
13736 
13737 // This pattern is automatically generated from aarch64_ad.m4
13738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13739 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13740 %{
13741   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13742   ins_cost(1.9 * INSN_COST);
13743   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13744 
13745    ins_encode %{
13746      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13747             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13748    %}
13749   ins_pipe(ialu_reg_reg_shift);
13750 %}
13751 
13752 // This pattern is automatically generated from aarch64_ad.m4
13753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13754 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13755 %{
13756   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13757   ins_cost(1.9 * INSN_COST);
13758   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13759 
13760    ins_encode %{
13761      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13762             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13763    %}
13764   ins_pipe(ialu_reg_reg_shift);
13765 %}
13766 
13767 // This pattern is automatically generated from aarch64_ad.m4
13768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13769 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13770 %{
13771   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13772   ins_cost(1.9 * INSN_COST);
13773   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13774 
13775    ins_encode %{
13776      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13777             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13778    %}
13779   ins_pipe(ialu_reg_reg_shift);
13780 %}
13781 
13782 // This pattern is automatically generated from aarch64_ad.m4
13783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13784 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13785 %{
13786   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13787   ins_cost(1.9 * INSN_COST);
13788   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13789 
13790    ins_encode %{
13791      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13792             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13793    %}
13794   ins_pipe(ialu_reg_reg_shift);
13795 %}
13796 
13797 
13798 
13799 // END This section of the file is automatically generated. Do not edit --------------
13800 
13801 
13802 // ============================================================================
13803 // Floating Point Arithmetic Instructions
13804 
13805 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13806   match(Set dst (AddF src1 src2));
13807 
13808   ins_cost(INSN_COST * 5);
13809   format %{ "fadds   $dst, $src1, $src2" %}
13810 
13811   ins_encode %{
13812     __ fadds(as_FloatRegister($dst$$reg),
13813              as_FloatRegister($src1$$reg),
13814              as_FloatRegister($src2$$reg));
13815   %}
13816 
13817   ins_pipe(fp_dop_reg_reg_s);
13818 %}
13819 
13820 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13821   match(Set dst (AddD src1 src2));
13822 
13823   ins_cost(INSN_COST * 5);
13824   format %{ "faddd   $dst, $src1, $src2" %}
13825 
13826   ins_encode %{
13827     __ faddd(as_FloatRegister($dst$$reg),
13828              as_FloatRegister($src1$$reg),
13829              as_FloatRegister($src2$$reg));
13830   %}
13831 
13832   ins_pipe(fp_dop_reg_reg_d);
13833 %}
13834 
13835 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13836   match(Set dst (SubF src1 src2));
13837 
13838   ins_cost(INSN_COST * 5);
13839   format %{ "fsubs   $dst, $src1, $src2" %}
13840 
13841   ins_encode %{
13842     __ fsubs(as_FloatRegister($dst$$reg),
13843              as_FloatRegister($src1$$reg),
13844              as_FloatRegister($src2$$reg));
13845   %}
13846 
13847   ins_pipe(fp_dop_reg_reg_s);
13848 %}
13849 
13850 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13851   match(Set dst (SubD src1 src2));
13852 
13853   ins_cost(INSN_COST * 5);
13854   format %{ "fsubd   $dst, $src1, $src2" %}
13855 
13856   ins_encode %{
13857     __ fsubd(as_FloatRegister($dst$$reg),
13858              as_FloatRegister($src1$$reg),
13859              as_FloatRegister($src2$$reg));
13860   %}
13861 
13862   ins_pipe(fp_dop_reg_reg_d);
13863 %}
13864 
13865 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13866   match(Set dst (MulF src1 src2));
13867 
13868   ins_cost(INSN_COST * 6);
13869   format %{ "fmuls   $dst, $src1, $src2" %}
13870 
13871   ins_encode %{
13872     __ fmuls(as_FloatRegister($dst$$reg),
13873              as_FloatRegister($src1$$reg),
13874              as_FloatRegister($src2$$reg));
13875   %}
13876 
13877   ins_pipe(fp_dop_reg_reg_s);
13878 %}
13879 
13880 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13881   match(Set dst (MulD src1 src2));
13882 
13883   ins_cost(INSN_COST * 6);
13884   format %{ "fmuld   $dst, $src1, $src2" %}
13885 
13886   ins_encode %{
13887     __ fmuld(as_FloatRegister($dst$$reg),
13888              as_FloatRegister($src1$$reg),
13889              as_FloatRegister($src2$$reg));
13890   %}
13891 
13892   ins_pipe(fp_dop_reg_reg_d);
13893 %}
13894 
13895 // src1 * src2 + src3
13896 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13897   predicate(UseFMA);
13898   match(Set dst (FmaF src3 (Binary src1 src2)));
13899 
13900   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13901 
13902   ins_encode %{
13903     __ fmadds(as_FloatRegister($dst$$reg),
13904              as_FloatRegister($src1$$reg),
13905              as_FloatRegister($src2$$reg),
13906              as_FloatRegister($src3$$reg));
13907   %}
13908 
13909   ins_pipe(pipe_class_default);
13910 %}
13911 
13912 // src1 * src2 + src3
13913 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13914   predicate(UseFMA);
13915   match(Set dst (FmaD src3 (Binary src1 src2)));
13916 
13917   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13918 
13919   ins_encode %{
13920     __ fmaddd(as_FloatRegister($dst$$reg),
13921              as_FloatRegister($src1$$reg),
13922              as_FloatRegister($src2$$reg),
13923              as_FloatRegister($src3$$reg));
13924   %}
13925 
13926   ins_pipe(pipe_class_default);
13927 %}
13928 
13929 // -src1 * src2 + src3
13930 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13931   predicate(UseFMA);
13932   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13933   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13934 
13935   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13936 
13937   ins_encode %{
13938     __ fmsubs(as_FloatRegister($dst$$reg),
13939               as_FloatRegister($src1$$reg),
13940               as_FloatRegister($src2$$reg),
13941               as_FloatRegister($src3$$reg));
13942   %}
13943 
13944   ins_pipe(pipe_class_default);
13945 %}
13946 
13947 // -src1 * src2 + src3
13948 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13949   predicate(UseFMA);
13950   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13951   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13952 
13953   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13954 
13955   ins_encode %{
13956     __ fmsubd(as_FloatRegister($dst$$reg),
13957               as_FloatRegister($src1$$reg),
13958               as_FloatRegister($src2$$reg),
13959               as_FloatRegister($src3$$reg));
13960   %}
13961 
13962   ins_pipe(pipe_class_default);
13963 %}
13964 
13965 // -src1 * src2 - src3
13966 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13967   predicate(UseFMA);
13968   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13969   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13970 
13971   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13972 
13973   ins_encode %{
13974     __ fnmadds(as_FloatRegister($dst$$reg),
13975                as_FloatRegister($src1$$reg),
13976                as_FloatRegister($src2$$reg),
13977                as_FloatRegister($src3$$reg));
13978   %}
13979 
13980   ins_pipe(pipe_class_default);
13981 %}
13982 
13983 // -src1 * src2 - src3
13984 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13985   predicate(UseFMA);
13986   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13987   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13988 
13989   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13990 
13991   ins_encode %{
13992     __ fnmaddd(as_FloatRegister($dst$$reg),
13993                as_FloatRegister($src1$$reg),
13994                as_FloatRegister($src2$$reg),
13995                as_FloatRegister($src3$$reg));
13996   %}
13997 
13998   ins_pipe(pipe_class_default);
13999 %}
14000 
14001 // src1 * src2 - src3
14002 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14003   predicate(UseFMA);
14004   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14005 
14006   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14007 
14008   ins_encode %{
14009     __ fnmsubs(as_FloatRegister($dst$$reg),
14010                as_FloatRegister($src1$$reg),
14011                as_FloatRegister($src2$$reg),
14012                as_FloatRegister($src3$$reg));
14013   %}
14014 
14015   ins_pipe(pipe_class_default);
14016 %}
14017 
14018 // src1 * src2 - src3
14019 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14020   predicate(UseFMA);
14021   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14022 
14023   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14024 
14025   ins_encode %{
14026   // n.b. insn name should be fnmsubd
14027     __ fnmsub(as_FloatRegister($dst$$reg),
14028               as_FloatRegister($src1$$reg),
14029               as_FloatRegister($src2$$reg),
14030               as_FloatRegister($src3$$reg));
14031   %}
14032 
14033   ins_pipe(pipe_class_default);
14034 %}
14035 
14036 
14037 // Math.max(FF)F
14038 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14039   match(Set dst (MaxF src1 src2));
14040 
14041   format %{ "fmaxs   $dst, $src1, $src2" %}
14042   ins_encode %{
14043     __ fmaxs(as_FloatRegister($dst$$reg),
14044              as_FloatRegister($src1$$reg),
14045              as_FloatRegister($src2$$reg));
14046   %}
14047 
14048   ins_pipe(fp_dop_reg_reg_s);
14049 %}
14050 
14051 // Math.min(FF)F
14052 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14053   match(Set dst (MinF src1 src2));
14054 
14055   format %{ "fmins   $dst, $src1, $src2" %}
14056   ins_encode %{
14057     __ fmins(as_FloatRegister($dst$$reg),
14058              as_FloatRegister($src1$$reg),
14059              as_FloatRegister($src2$$reg));
14060   %}
14061 
14062   ins_pipe(fp_dop_reg_reg_s);
14063 %}
14064 
14065 // Math.max(DD)D
14066 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14067   match(Set dst (MaxD src1 src2));
14068 
14069   format %{ "fmaxd   $dst, $src1, $src2" %}
14070   ins_encode %{
14071     __ fmaxd(as_FloatRegister($dst$$reg),
14072              as_FloatRegister($src1$$reg),
14073              as_FloatRegister($src2$$reg));
14074   %}
14075 
14076   ins_pipe(fp_dop_reg_reg_d);
14077 %}
14078 
14079 // Math.min(DD)D
14080 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14081   match(Set dst (MinD src1 src2));
14082 
14083   format %{ "fmind   $dst, $src1, $src2" %}
14084   ins_encode %{
14085     __ fmind(as_FloatRegister($dst$$reg),
14086              as_FloatRegister($src1$$reg),
14087              as_FloatRegister($src2$$reg));
14088   %}
14089 
14090   ins_pipe(fp_dop_reg_reg_d);
14091 %}
14092 
14093 
14094 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14095   match(Set dst (DivF src1  src2));
14096 
14097   ins_cost(INSN_COST * 18);
14098   format %{ "fdivs   $dst, $src1, $src2" %}
14099 
14100   ins_encode %{
14101     __ fdivs(as_FloatRegister($dst$$reg),
14102              as_FloatRegister($src1$$reg),
14103              as_FloatRegister($src2$$reg));
14104   %}
14105 
14106   ins_pipe(fp_div_s);
14107 %}
14108 
14109 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14110   match(Set dst (DivD src1  src2));
14111 
14112   ins_cost(INSN_COST * 32);
14113   format %{ "fdivd   $dst, $src1, $src2" %}
14114 
14115   ins_encode %{
14116     __ fdivd(as_FloatRegister($dst$$reg),
14117              as_FloatRegister($src1$$reg),
14118              as_FloatRegister($src2$$reg));
14119   %}
14120 
14121   ins_pipe(fp_div_d);
14122 %}
14123 
14124 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14125   match(Set dst (NegF src));
14126 
14127   ins_cost(INSN_COST * 3);
14128   format %{ "fneg   $dst, $src" %}
14129 
14130   ins_encode %{
14131     __ fnegs(as_FloatRegister($dst$$reg),
14132              as_FloatRegister($src$$reg));
14133   %}
14134 
14135   ins_pipe(fp_uop_s);
14136 %}
14137 
14138 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14139   match(Set dst (NegD src));
14140 
14141   ins_cost(INSN_COST * 3);
14142   format %{ "fnegd   $dst, $src" %}
14143 
14144   ins_encode %{
14145     __ fnegd(as_FloatRegister($dst$$reg),
14146              as_FloatRegister($src$$reg));
14147   %}
14148 
14149   ins_pipe(fp_uop_d);
14150 %}
14151 
14152 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14153 %{
14154   match(Set dst (AbsI src));
14155 
14156   effect(KILL cr);
14157   ins_cost(INSN_COST * 2);
14158   format %{ "cmpw  $src, zr\n\t"
14159             "cnegw $dst, $src, Assembler::LT\t# int abs"
14160   %}
14161 
14162   ins_encode %{
14163     __ cmpw(as_Register($src$$reg), zr);
14164     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14165   %}
14166   ins_pipe(pipe_class_default);
14167 %}
14168 
14169 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14170 %{
14171   match(Set dst (AbsL src));
14172 
14173   effect(KILL cr);
14174   ins_cost(INSN_COST * 2);
14175   format %{ "cmp  $src, zr\n\t"
14176             "cneg $dst, $src, Assembler::LT\t# long abs"
14177   %}
14178 
14179   ins_encode %{
14180     __ cmp(as_Register($src$$reg), zr);
14181     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14182   %}
14183   ins_pipe(pipe_class_default);
14184 %}
14185 
14186 instruct absF_reg(vRegF dst, vRegF src) %{
14187   match(Set dst (AbsF src));
14188 
14189   ins_cost(INSN_COST * 3);
14190   format %{ "fabss   $dst, $src" %}
14191   ins_encode %{
14192     __ fabss(as_FloatRegister($dst$$reg),
14193              as_FloatRegister($src$$reg));
14194   %}
14195 
14196   ins_pipe(fp_uop_s);
14197 %}
14198 
14199 instruct absD_reg(vRegD dst, vRegD src) %{
14200   match(Set dst (AbsD src));
14201 
14202   ins_cost(INSN_COST * 3);
14203   format %{ "fabsd   $dst, $src" %}
14204   ins_encode %{
14205     __ fabsd(as_FloatRegister($dst$$reg),
14206              as_FloatRegister($src$$reg));
14207   %}
14208 
14209   ins_pipe(fp_uop_d);
14210 %}
14211 
14212 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14213   match(Set dst (AbsF (SubF src1 src2)));
14214 
14215   ins_cost(INSN_COST * 3);
14216   format %{ "fabds   $dst, $src1, $src2" %}
14217   ins_encode %{
14218     __ fabds(as_FloatRegister($dst$$reg),
14219              as_FloatRegister($src1$$reg),
14220              as_FloatRegister($src2$$reg));
14221   %}
14222 
14223   ins_pipe(fp_uop_s);
14224 %}
14225 
14226 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14227   match(Set dst (AbsD (SubD src1 src2)));
14228 
14229   ins_cost(INSN_COST * 3);
14230   format %{ "fabdd   $dst, $src1, $src2" %}
14231   ins_encode %{
14232     __ fabdd(as_FloatRegister($dst$$reg),
14233              as_FloatRegister($src1$$reg),
14234              as_FloatRegister($src2$$reg));
14235   %}
14236 
14237   ins_pipe(fp_uop_d);
14238 %}
14239 
14240 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14241   match(Set dst (SqrtD src));
14242 
14243   ins_cost(INSN_COST * 50);
14244   format %{ "fsqrtd  $dst, $src" %}
14245   ins_encode %{
14246     __ fsqrtd(as_FloatRegister($dst$$reg),
14247              as_FloatRegister($src$$reg));
14248   %}
14249 
14250   ins_pipe(fp_div_s);
14251 %}
14252 
14253 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14254   match(Set dst (SqrtF src));
14255 
14256   ins_cost(INSN_COST * 50);
14257   format %{ "fsqrts  $dst, $src" %}
14258   ins_encode %{
14259     __ fsqrts(as_FloatRegister($dst$$reg),
14260              as_FloatRegister($src$$reg));
14261   %}
14262 
14263   ins_pipe(fp_div_d);
14264 %}
14265 
14266 // Math.rint, floor, ceil
14267 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14268   match(Set dst (RoundDoubleMode src rmode));
14269   format %{ "frint  $dst, $src, $rmode" %}
14270   ins_encode %{
14271     switch ($rmode$$constant) {
14272       case RoundDoubleModeNode::rmode_rint:
14273         __ frintnd(as_FloatRegister($dst$$reg),
14274                    as_FloatRegister($src$$reg));
14275         break;
14276       case RoundDoubleModeNode::rmode_floor:
14277         __ frintmd(as_FloatRegister($dst$$reg),
14278                    as_FloatRegister($src$$reg));
14279         break;
14280       case RoundDoubleModeNode::rmode_ceil:
14281         __ frintpd(as_FloatRegister($dst$$reg),
14282                    as_FloatRegister($src$$reg));
14283         break;
14284     }
14285   %}
14286   ins_pipe(fp_uop_d);
14287 %}
14288 
14289 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14290   match(Set dst (CopySignD src1 (Binary src2 zero)));
14291   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14292   format %{ "CopySignD  $dst $src1 $src2" %}
14293   ins_encode %{
14294     FloatRegister dst = as_FloatRegister($dst$$reg),
14295                   src1 = as_FloatRegister($src1$$reg),
14296                   src2 = as_FloatRegister($src2$$reg),
14297                   zero = as_FloatRegister($zero$$reg);
14298     __ fnegd(dst, zero);
14299     __ bsl(dst, __ T8B, src2, src1);
14300   %}
14301   ins_pipe(fp_uop_d);
14302 %}
14303 
14304 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14305   match(Set dst (CopySignF src1 src2));
14306   effect(TEMP_DEF dst, USE src1, USE src2);
14307   format %{ "CopySignF  $dst $src1 $src2" %}
14308   ins_encode %{
14309     FloatRegister dst = as_FloatRegister($dst$$reg),
14310                   src1 = as_FloatRegister($src1$$reg),
14311                   src2 = as_FloatRegister($src2$$reg);
14312     __ movi(dst, __ T2S, 0x80, 24);
14313     __ bsl(dst, __ T8B, src2, src1);
14314   %}
14315   ins_pipe(fp_uop_d);
14316 %}
14317 
14318 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14319   match(Set dst (SignumD src (Binary zero one)));
14320   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14321   format %{ "signumD  $dst, $src" %}
14322   ins_encode %{
14323     FloatRegister src = as_FloatRegister($src$$reg),
14324                   dst = as_FloatRegister($dst$$reg),
14325                   zero = as_FloatRegister($zero$$reg),
14326                   one = as_FloatRegister($one$$reg);
14327     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14328     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14329     // Bit selection instruction gets bit from "one" for each enabled bit in
14330     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14331     // NaN the whole "src" will be copied because "dst" is zero. For all other
14332     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14333     // from "src", and all other bits are copied from 1.0.
14334     __ bsl(dst, __ T8B, one, src);
14335   %}
14336   ins_pipe(fp_uop_d);
14337 %}
14338 
14339 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14340   match(Set dst (SignumF src (Binary zero one)));
14341   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14342   format %{ "signumF  $dst, $src" %}
14343   ins_encode %{
14344     FloatRegister src = as_FloatRegister($src$$reg),
14345                   dst = as_FloatRegister($dst$$reg),
14346                   zero = as_FloatRegister($zero$$reg),
14347                   one = as_FloatRegister($one$$reg);
14348     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14349     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14350     // Bit selection instruction gets bit from "one" for each enabled bit in
14351     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14352     // NaN the whole "src" will be copied because "dst" is zero. For all other
14353     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14354     // from "src", and all other bits are copied from 1.0.
14355     __ bsl(dst, __ T8B, one, src);
14356   %}
14357   ins_pipe(fp_uop_d);
14358 %}
14359 
14360 // ============================================================================
14361 // Logical Instructions
14362 
14363 // Integer Logical Instructions
14364 
14365 // And Instructions
14366 
14367 
14368 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14369   match(Set dst (AndI src1 src2));
14370 
14371   format %{ "andw  $dst, $src1, $src2\t# int" %}
14372 
14373   ins_cost(INSN_COST);
14374   ins_encode %{
14375     __ andw(as_Register($dst$$reg),
14376             as_Register($src1$$reg),
14377             as_Register($src2$$reg));
14378   %}
14379 
14380   ins_pipe(ialu_reg_reg);
14381 %}
14382 
14383 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14384   match(Set dst (AndI src1 src2));
14385 
14386   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14387 
14388   ins_cost(INSN_COST);
14389   ins_encode %{
14390     __ andw(as_Register($dst$$reg),
14391             as_Register($src1$$reg),
14392             (uint64_t)($src2$$constant));
14393   %}
14394 
14395   ins_pipe(ialu_reg_imm);
14396 %}
14397 
14398 // Or Instructions
14399 
14400 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14401   match(Set dst (OrI src1 src2));
14402 
14403   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14404 
14405   ins_cost(INSN_COST);
14406   ins_encode %{
14407     __ orrw(as_Register($dst$$reg),
14408             as_Register($src1$$reg),
14409             as_Register($src2$$reg));
14410   %}
14411 
14412   ins_pipe(ialu_reg_reg);
14413 %}
14414 
14415 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14416   match(Set dst (OrI src1 src2));
14417 
14418   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14419 
14420   ins_cost(INSN_COST);
14421   ins_encode %{
14422     __ orrw(as_Register($dst$$reg),
14423             as_Register($src1$$reg),
14424             (uint64_t)($src2$$constant));
14425   %}
14426 
14427   ins_pipe(ialu_reg_imm);
14428 %}
14429 
14430 // Xor Instructions
14431 
14432 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14433   match(Set dst (XorI src1 src2));
14434 
14435   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14436 
14437   ins_cost(INSN_COST);
14438   ins_encode %{
14439     __ eorw(as_Register($dst$$reg),
14440             as_Register($src1$$reg),
14441             as_Register($src2$$reg));
14442   %}
14443 
14444   ins_pipe(ialu_reg_reg);
14445 %}
14446 
14447 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14448   match(Set dst (XorI src1 src2));
14449 
14450   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14451 
14452   ins_cost(INSN_COST);
14453   ins_encode %{
14454     __ eorw(as_Register($dst$$reg),
14455             as_Register($src1$$reg),
14456             (uint64_t)($src2$$constant));
14457   %}
14458 
14459   ins_pipe(ialu_reg_imm);
14460 %}
14461 
14462 // Long Logical Instructions
14463 // TODO
14464 
14465 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14466   match(Set dst (AndL src1 src2));
14467 
14468   format %{ "and  $dst, $src1, $src2\t# int" %}
14469 
14470   ins_cost(INSN_COST);
14471   ins_encode %{
14472     __ andr(as_Register($dst$$reg),
14473             as_Register($src1$$reg),
14474             as_Register($src2$$reg));
14475   %}
14476 
14477   ins_pipe(ialu_reg_reg);
14478 %}
14479 
14480 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14481   match(Set dst (AndL src1 src2));
14482 
14483   format %{ "and  $dst, $src1, $src2\t# int" %}
14484 
14485   ins_cost(INSN_COST);
14486   ins_encode %{
14487     __ andr(as_Register($dst$$reg),
14488             as_Register($src1$$reg),
14489             (uint64_t)($src2$$constant));
14490   %}
14491 
14492   ins_pipe(ialu_reg_imm);
14493 %}
14494 
14495 // Or Instructions
14496 
14497 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14498   match(Set dst (OrL src1 src2));
14499 
14500   format %{ "orr  $dst, $src1, $src2\t# int" %}
14501 
14502   ins_cost(INSN_COST);
14503   ins_encode %{
14504     __ orr(as_Register($dst$$reg),
14505            as_Register($src1$$reg),
14506            as_Register($src2$$reg));
14507   %}
14508 
14509   ins_pipe(ialu_reg_reg);
14510 %}
14511 
14512 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14513   match(Set dst (OrL src1 src2));
14514 
14515   format %{ "orr  $dst, $src1, $src2\t# int" %}
14516 
14517   ins_cost(INSN_COST);
14518   ins_encode %{
14519     __ orr(as_Register($dst$$reg),
14520            as_Register($src1$$reg),
14521            (uint64_t)($src2$$constant));
14522   %}
14523 
14524   ins_pipe(ialu_reg_imm);
14525 %}
14526 
14527 // Xor Instructions
14528 
14529 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14530   match(Set dst (XorL src1 src2));
14531 
14532   format %{ "eor  $dst, $src1, $src2\t# int" %}
14533 
14534   ins_cost(INSN_COST);
14535   ins_encode %{
14536     __ eor(as_Register($dst$$reg),
14537            as_Register($src1$$reg),
14538            as_Register($src2$$reg));
14539   %}
14540 
14541   ins_pipe(ialu_reg_reg);
14542 %}
14543 
14544 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14545   match(Set dst (XorL src1 src2));
14546 
14547   ins_cost(INSN_COST);
14548   format %{ "eor  $dst, $src1, $src2\t# int" %}
14549 
14550   ins_encode %{
14551     __ eor(as_Register($dst$$reg),
14552            as_Register($src1$$reg),
14553            (uint64_t)($src2$$constant));
14554   %}
14555 
14556   ins_pipe(ialu_reg_imm);
14557 %}
14558 
14559 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14560 %{
14561   match(Set dst (ConvI2L src));
14562 
14563   ins_cost(INSN_COST);
14564   format %{ "sxtw  $dst, $src\t# i2l" %}
14565   ins_encode %{
14566     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14567   %}
14568   ins_pipe(ialu_reg_shift);
14569 %}
14570 
14571 // this pattern occurs in bigmath arithmetic
14572 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14573 %{
14574   match(Set dst (AndL (ConvI2L src) mask));
14575 
14576   ins_cost(INSN_COST);
14577   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14578   ins_encode %{
14579     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14580   %}
14581 
14582   ins_pipe(ialu_reg_shift);
14583 %}
14584 
14585 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14586   match(Set dst (ConvL2I src));
14587 
14588   ins_cost(INSN_COST);
14589   format %{ "movw  $dst, $src \t// l2i" %}
14590 
14591   ins_encode %{
14592     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14593   %}
14594 
14595   ins_pipe(ialu_reg);
14596 %}
14597 
14598 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14599 %{
14600   match(Set dst (Conv2B src));
14601   effect(KILL cr);
14602 
14603   format %{
14604     "cmpw $src, zr\n\t"
14605     "cset $dst, ne"
14606   %}
14607 
14608   ins_encode %{
14609     __ cmpw(as_Register($src$$reg), zr);
14610     __ cset(as_Register($dst$$reg), Assembler::NE);
14611   %}
14612 
14613   ins_pipe(ialu_reg);
14614 %}
14615 
14616 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14617 %{
14618   match(Set dst (Conv2B src));
14619   effect(KILL cr);
14620 
14621   format %{
14622     "cmp  $src, zr\n\t"
14623     "cset $dst, ne"
14624   %}
14625 
14626   ins_encode %{
14627     __ cmp(as_Register($src$$reg), zr);
14628     __ cset(as_Register($dst$$reg), Assembler::NE);
14629   %}
14630 
14631   ins_pipe(ialu_reg);
14632 %}
14633 
14634 instruct convD2F_reg(vRegF dst, vRegD src) %{
14635   match(Set dst (ConvD2F src));
14636 
14637   ins_cost(INSN_COST * 5);
14638   format %{ "fcvtd  $dst, $src \t// d2f" %}
14639 
14640   ins_encode %{
14641     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14642   %}
14643 
14644   ins_pipe(fp_d2f);
14645 %}
14646 
14647 instruct convF2D_reg(vRegD dst, vRegF src) %{
14648   match(Set dst (ConvF2D src));
14649 
14650   ins_cost(INSN_COST * 5);
14651   format %{ "fcvts  $dst, $src \t// f2d" %}
14652 
14653   ins_encode %{
14654     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14655   %}
14656 
14657   ins_pipe(fp_f2d);
14658 %}
14659 
14660 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14661   match(Set dst (ConvF2I src));
14662 
14663   ins_cost(INSN_COST * 5);
14664   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14665 
14666   ins_encode %{
14667     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14668   %}
14669 
14670   ins_pipe(fp_f2i);
14671 %}
14672 
14673 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14674   match(Set dst (ConvF2L src));
14675 
14676   ins_cost(INSN_COST * 5);
14677   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14678 
14679   ins_encode %{
14680     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14681   %}
14682 
14683   ins_pipe(fp_f2l);
14684 %}
14685 
14686 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14687   match(Set dst (ConvI2F src));
14688 
14689   ins_cost(INSN_COST * 5);
14690   format %{ "scvtfws  $dst, $src \t// i2f" %}
14691 
14692   ins_encode %{
14693     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14694   %}
14695 
14696   ins_pipe(fp_i2f);
14697 %}
14698 
14699 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14700   match(Set dst (ConvL2F src));
14701 
14702   ins_cost(INSN_COST * 5);
14703   format %{ "scvtfs  $dst, $src \t// l2f" %}
14704 
14705   ins_encode %{
14706     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14707   %}
14708 
14709   ins_pipe(fp_l2f);
14710 %}
14711 
14712 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14713   match(Set dst (ConvD2I src));
14714 
14715   ins_cost(INSN_COST * 5);
14716   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14717 
14718   ins_encode %{
14719     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14720   %}
14721 
14722   ins_pipe(fp_d2i);
14723 %}
14724 
14725 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14726   match(Set dst (ConvD2L src));
14727 
14728   ins_cost(INSN_COST * 5);
14729   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14730 
14731   ins_encode %{
14732     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14733   %}
14734 
14735   ins_pipe(fp_d2l);
14736 %}
14737 
14738 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14739   match(Set dst (ConvI2D src));
14740 
14741   ins_cost(INSN_COST * 5);
14742   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14743 
14744   ins_encode %{
14745     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14746   %}
14747 
14748   ins_pipe(fp_i2d);
14749 %}
14750 
14751 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14752   match(Set dst (ConvL2D src));
14753 
14754   ins_cost(INSN_COST * 5);
14755   format %{ "scvtfd  $dst, $src \t// l2d" %}
14756 
14757   ins_encode %{
14758     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14759   %}
14760 
14761   ins_pipe(fp_l2d);
14762 %}
14763 
14764 // stack <-> reg and reg <-> reg shuffles with no conversion
14765 
14766 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14767 
14768   match(Set dst (MoveF2I src));
14769 
14770   effect(DEF dst, USE src);
14771 
14772   ins_cost(4 * INSN_COST);
14773 
14774   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14775 
14776   ins_encode %{
14777     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14778   %}
14779 
14780   ins_pipe(iload_reg_reg);
14781 
14782 %}
14783 
14784 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14785 
14786   match(Set dst (MoveI2F src));
14787 
14788   effect(DEF dst, USE src);
14789 
14790   ins_cost(4 * INSN_COST);
14791 
14792   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14793 
14794   ins_encode %{
14795     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14796   %}
14797 
14798   ins_pipe(pipe_class_memory);
14799 
14800 %}
14801 
14802 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14803 
14804   match(Set dst (MoveD2L src));
14805 
14806   effect(DEF dst, USE src);
14807 
14808   ins_cost(4 * INSN_COST);
14809 
14810   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14811 
14812   ins_encode %{
14813     __ ldr($dst$$Register, Address(sp, $src$$disp));
14814   %}
14815 
14816   ins_pipe(iload_reg_reg);
14817 
14818 %}
14819 
14820 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14821 
14822   match(Set dst (MoveL2D src));
14823 
14824   effect(DEF dst, USE src);
14825 
14826   ins_cost(4 * INSN_COST);
14827 
14828   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14829 
14830   ins_encode %{
14831     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14832   %}
14833 
14834   ins_pipe(pipe_class_memory);
14835 
14836 %}
14837 
14838 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14839 
14840   match(Set dst (MoveF2I src));
14841 
14842   effect(DEF dst, USE src);
14843 
14844   ins_cost(INSN_COST);
14845 
14846   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14847 
14848   ins_encode %{
14849     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14850   %}
14851 
14852   ins_pipe(pipe_class_memory);
14853 
14854 %}
14855 
14856 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14857 
14858   match(Set dst (MoveI2F src));
14859 
14860   effect(DEF dst, USE src);
14861 
14862   ins_cost(INSN_COST);
14863 
14864   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14865 
14866   ins_encode %{
14867     __ strw($src$$Register, Address(sp, $dst$$disp));
14868   %}
14869 
14870   ins_pipe(istore_reg_reg);
14871 
14872 %}
14873 
14874 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14875 
14876   match(Set dst (MoveD2L src));
14877 
14878   effect(DEF dst, USE src);
14879 
14880   ins_cost(INSN_COST);
14881 
14882   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14883 
14884   ins_encode %{
14885     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14886   %}
14887 
14888   ins_pipe(pipe_class_memory);
14889 
14890 %}
14891 
14892 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14893 
14894   match(Set dst (MoveL2D src));
14895 
14896   effect(DEF dst, USE src);
14897 
14898   ins_cost(INSN_COST);
14899 
14900   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14901 
14902   ins_encode %{
14903     __ str($src$$Register, Address(sp, $dst$$disp));
14904   %}
14905 
14906   ins_pipe(istore_reg_reg);
14907 
14908 %}
14909 
14910 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14911 
14912   match(Set dst (MoveF2I src));
14913 
14914   effect(DEF dst, USE src);
14915 
14916   ins_cost(INSN_COST);
14917 
14918   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14919 
14920   ins_encode %{
14921     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14922   %}
14923 
14924   ins_pipe(fp_f2i);
14925 
14926 %}
14927 
14928 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14929 
14930   match(Set dst (MoveI2F src));
14931 
14932   effect(DEF dst, USE src);
14933 
14934   ins_cost(INSN_COST);
14935 
14936   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14937 
14938   ins_encode %{
14939     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14940   %}
14941 
14942   ins_pipe(fp_i2f);
14943 
14944 %}
14945 
14946 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14947 
14948   match(Set dst (MoveD2L src));
14949 
14950   effect(DEF dst, USE src);
14951 
14952   ins_cost(INSN_COST);
14953 
14954   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14955 
14956   ins_encode %{
14957     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14958   %}
14959 
14960   ins_pipe(fp_d2l);
14961 
14962 %}
14963 
14964 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14965 
14966   match(Set dst (MoveL2D src));
14967 
14968   effect(DEF dst, USE src);
14969 
14970   ins_cost(INSN_COST);
14971 
14972   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14973 
14974   ins_encode %{
14975     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14976   %}
14977 
14978   ins_pipe(fp_l2d);
14979 
14980 %}
14981 
14982 // ============================================================================
14983 // clearing of an array
14984 
14985 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14986 %{
14987   match(Set dummy (ClearArray cnt base));
14988   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14989 
14990   ins_cost(4 * INSN_COST);
14991   format %{ "ClearArray $cnt, $base" %}
14992 
14993   ins_encode %{
14994     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14995     if (tpc == NULL) {
14996       ciEnv::current()->record_failure("CodeCache is full");
14997       return;
14998     }
14999   %}
15000 
15001   ins_pipe(pipe_class_memory);
15002 %}
15003 
15004 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15005 %{
15006   predicate((uint64_t)n->in(2)->get_long()
15007             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15008   match(Set dummy (ClearArray cnt base));
15009   effect(TEMP temp, USE_KILL base, KILL cr);
15010 
15011   ins_cost(4 * INSN_COST);
15012   format %{ "ClearArray $cnt, $base" %}
15013 
15014   ins_encode %{
15015     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15016   %}
15017 
15018   ins_pipe(pipe_class_memory);
15019 %}
15020 
15021 // ============================================================================
15022 // Overflow Math Instructions
15023 
15024 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15025 %{
15026   match(Set cr (OverflowAddI op1 op2));
15027 
15028   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15029   ins_cost(INSN_COST);
15030   ins_encode %{
15031     __ cmnw($op1$$Register, $op2$$Register);
15032   %}
15033 
15034   ins_pipe(icmp_reg_reg);
15035 %}
15036 
15037 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15038 %{
15039   match(Set cr (OverflowAddI op1 op2));
15040 
15041   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15042   ins_cost(INSN_COST);
15043   ins_encode %{
15044     __ cmnw($op1$$Register, $op2$$constant);
15045   %}
15046 
15047   ins_pipe(icmp_reg_imm);
15048 %}
15049 
15050 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15051 %{
15052   match(Set cr (OverflowAddL op1 op2));
15053 
15054   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15055   ins_cost(INSN_COST);
15056   ins_encode %{
15057     __ cmn($op1$$Register, $op2$$Register);
15058   %}
15059 
15060   ins_pipe(icmp_reg_reg);
15061 %}
15062 
15063 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15064 %{
15065   match(Set cr (OverflowAddL op1 op2));
15066 
15067   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15068   ins_cost(INSN_COST);
15069   ins_encode %{
15070     __ cmn($op1$$Register, $op2$$constant);
15071   %}
15072 
15073   ins_pipe(icmp_reg_imm);
15074 %}
15075 
15076 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15077 %{
15078   match(Set cr (OverflowSubI op1 op2));
15079 
15080   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15081   ins_cost(INSN_COST);
15082   ins_encode %{
15083     __ cmpw($op1$$Register, $op2$$Register);
15084   %}
15085 
15086   ins_pipe(icmp_reg_reg);
15087 %}
15088 
15089 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15090 %{
15091   match(Set cr (OverflowSubI op1 op2));
15092 
15093   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15094   ins_cost(INSN_COST);
15095   ins_encode %{
15096     __ cmpw($op1$$Register, $op2$$constant);
15097   %}
15098 
15099   ins_pipe(icmp_reg_imm);
15100 %}
15101 
15102 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15103 %{
15104   match(Set cr (OverflowSubL op1 op2));
15105 
15106   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15107   ins_cost(INSN_COST);
15108   ins_encode %{
15109     __ cmp($op1$$Register, $op2$$Register);
15110   %}
15111 
15112   ins_pipe(icmp_reg_reg);
15113 %}
15114 
15115 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15116 %{
15117   match(Set cr (OverflowSubL op1 op2));
15118 
15119   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15120   ins_cost(INSN_COST);
15121   ins_encode %{
15122     __ subs(zr, $op1$$Register, $op2$$constant);
15123   %}
15124 
15125   ins_pipe(icmp_reg_imm);
15126 %}
15127 
15128 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15129 %{
15130   match(Set cr (OverflowSubI zero op1));
15131 
15132   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15133   ins_cost(INSN_COST);
15134   ins_encode %{
15135     __ cmpw(zr, $op1$$Register);
15136   %}
15137 
15138   ins_pipe(icmp_reg_imm);
15139 %}
15140 
15141 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15142 %{
15143   match(Set cr (OverflowSubL zero op1));
15144 
15145   format %{ "cmp   zr, $op1\t# overflow check long" %}
15146   ins_cost(INSN_COST);
15147   ins_encode %{
15148     __ cmp(zr, $op1$$Register);
15149   %}
15150 
15151   ins_pipe(icmp_reg_imm);
15152 %}
15153 
15154 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15155 %{
15156   match(Set cr (OverflowMulI op1 op2));
15157 
15158   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15159             "cmp   rscratch1, rscratch1, sxtw\n\t"
15160             "movw  rscratch1, #0x80000000\n\t"
15161             "cselw rscratch1, rscratch1, zr, NE\n\t"
15162             "cmpw  rscratch1, #1" %}
15163   ins_cost(5 * INSN_COST);
15164   ins_encode %{
15165     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15166     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15167     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15168     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15169     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15170   %}
15171 
15172   ins_pipe(pipe_slow);
15173 %}
15174 
15175 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15176 %{
15177   match(If cmp (OverflowMulI op1 op2));
15178   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15179             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15180   effect(USE labl, KILL cr);
15181 
15182   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15183             "cmp   rscratch1, rscratch1, sxtw\n\t"
15184             "b$cmp   $labl" %}
15185   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15186   ins_encode %{
15187     Label* L = $labl$$label;
15188     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15189     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15190     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15191     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15192   %}
15193 
15194   ins_pipe(pipe_serial);
15195 %}
15196 
15197 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15198 %{
15199   match(Set cr (OverflowMulL op1 op2));
15200 
15201   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15202             "smulh rscratch2, $op1, $op2\n\t"
15203             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15204             "movw  rscratch1, #0x80000000\n\t"
15205             "cselw rscratch1, rscratch1, zr, NE\n\t"
15206             "cmpw  rscratch1, #1" %}
15207   ins_cost(6 * INSN_COST);
15208   ins_encode %{
15209     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15210     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15211     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15212     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15213     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15214     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15215   %}
15216 
15217   ins_pipe(pipe_slow);
15218 %}
15219 
15220 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15221 %{
15222   match(If cmp (OverflowMulL op1 op2));
15223   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15224             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15225   effect(USE labl, KILL cr);
15226 
15227   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15228             "smulh rscratch2, $op1, $op2\n\t"
15229             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15230             "b$cmp $labl" %}
15231   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15232   ins_encode %{
15233     Label* L = $labl$$label;
15234     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15235     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15236     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15237     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15238     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15239   %}
15240 
15241   ins_pipe(pipe_serial);
15242 %}
15243 
15244 // ============================================================================
15245 // Compare Instructions
15246 
15247 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15248 %{
15249   match(Set cr (CmpI op1 op2));
15250 
15251   effect(DEF cr, USE op1, USE op2);
15252 
15253   ins_cost(INSN_COST);
15254   format %{ "cmpw  $op1, $op2" %}
15255 
15256   ins_encode(aarch64_enc_cmpw(op1, op2));
15257 
15258   ins_pipe(icmp_reg_reg);
15259 %}
15260 
15261 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15262 %{
15263   match(Set cr (CmpI op1 zero));
15264 
15265   effect(DEF cr, USE op1);
15266 
15267   ins_cost(INSN_COST);
15268   format %{ "cmpw $op1, 0" %}
15269 
15270   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15271 
15272   ins_pipe(icmp_reg_imm);
15273 %}
15274 
15275 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15276 %{
15277   match(Set cr (CmpI op1 op2));
15278 
15279   effect(DEF cr, USE op1);
15280 
15281   ins_cost(INSN_COST);
15282   format %{ "cmpw  $op1, $op2" %}
15283 
15284   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15285 
15286   ins_pipe(icmp_reg_imm);
15287 %}
15288 
15289 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15290 %{
15291   match(Set cr (CmpI op1 op2));
15292 
15293   effect(DEF cr, USE op1);
15294 
15295   ins_cost(INSN_COST * 2);
15296   format %{ "cmpw  $op1, $op2" %}
15297 
15298   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15299 
15300   ins_pipe(icmp_reg_imm);
15301 %}
15302 
15303 // Unsigned compare Instructions; really, same as signed compare
15304 // except it should only be used to feed an If or a CMovI which takes a
15305 // cmpOpU.
15306 
15307 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15308 %{
15309   match(Set cr (CmpU op1 op2));
15310 
15311   effect(DEF cr, USE op1, USE op2);
15312 
15313   ins_cost(INSN_COST);
15314   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15315 
15316   ins_encode(aarch64_enc_cmpw(op1, op2));
15317 
15318   ins_pipe(icmp_reg_reg);
15319 %}
15320 
15321 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15322 %{
15323   match(Set cr (CmpU op1 zero));
15324 
15325   effect(DEF cr, USE op1);
15326 
15327   ins_cost(INSN_COST);
15328   format %{ "cmpw $op1, #0\t# unsigned" %}
15329 
15330   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15331 
15332   ins_pipe(icmp_reg_imm);
15333 %}
15334 
15335 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15336 %{
15337   match(Set cr (CmpU op1 op2));
15338 
15339   effect(DEF cr, USE op1);
15340 
15341   ins_cost(INSN_COST);
15342   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15343 
15344   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15345 
15346   ins_pipe(icmp_reg_imm);
15347 %}
15348 
15349 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15350 %{
15351   match(Set cr (CmpU op1 op2));
15352 
15353   effect(DEF cr, USE op1);
15354 
15355   ins_cost(INSN_COST * 2);
15356   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15357 
15358   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15359 
15360   ins_pipe(icmp_reg_imm);
15361 %}
15362 
15363 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15364 %{
15365   match(Set cr (CmpL op1 op2));
15366 
15367   effect(DEF cr, USE op1, USE op2);
15368 
15369   ins_cost(INSN_COST);
15370   format %{ "cmp  $op1, $op2" %}
15371 
15372   ins_encode(aarch64_enc_cmp(op1, op2));
15373 
15374   ins_pipe(icmp_reg_reg);
15375 %}
15376 
15377 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15378 %{
15379   match(Set cr (CmpL op1 zero));
15380 
15381   effect(DEF cr, USE op1);
15382 
15383   ins_cost(INSN_COST);
15384   format %{ "tst  $op1" %}
15385 
15386   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15387 
15388   ins_pipe(icmp_reg_imm);
15389 %}
15390 
15391 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15392 %{
15393   match(Set cr (CmpL op1 op2));
15394 
15395   effect(DEF cr, USE op1);
15396 
15397   ins_cost(INSN_COST);
15398   format %{ "cmp  $op1, $op2" %}
15399 
15400   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15401 
15402   ins_pipe(icmp_reg_imm);
15403 %}
15404 
15405 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15406 %{
15407   match(Set cr (CmpL op1 op2));
15408 
15409   effect(DEF cr, USE op1);
15410 
15411   ins_cost(INSN_COST * 2);
15412   format %{ "cmp  $op1, $op2" %}
15413 
15414   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15415 
15416   ins_pipe(icmp_reg_imm);
15417 %}
15418 
15419 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15420 %{
15421   match(Set cr (CmpUL op1 op2));
15422 
15423   effect(DEF cr, USE op1, USE op2);
15424 
15425   ins_cost(INSN_COST);
15426   format %{ "cmp  $op1, $op2" %}
15427 
15428   ins_encode(aarch64_enc_cmp(op1, op2));
15429 
15430   ins_pipe(icmp_reg_reg);
15431 %}
15432 
15433 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15434 %{
15435   match(Set cr (CmpUL op1 zero));
15436 
15437   effect(DEF cr, USE op1);
15438 
15439   ins_cost(INSN_COST);
15440   format %{ "tst  $op1" %}
15441 
15442   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15443 
15444   ins_pipe(icmp_reg_imm);
15445 %}
15446 
15447 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15448 %{
15449   match(Set cr (CmpUL op1 op2));
15450 
15451   effect(DEF cr, USE op1);
15452 
15453   ins_cost(INSN_COST);
15454   format %{ "cmp  $op1, $op2" %}
15455 
15456   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15457 
15458   ins_pipe(icmp_reg_imm);
15459 %}
15460 
15461 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15462 %{
15463   match(Set cr (CmpUL op1 op2));
15464 
15465   effect(DEF cr, USE op1);
15466 
15467   ins_cost(INSN_COST * 2);
15468   format %{ "cmp  $op1, $op2" %}
15469 
15470   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15471 
15472   ins_pipe(icmp_reg_imm);
15473 %}
15474 
15475 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15476 %{
15477   match(Set cr (CmpP op1 op2));
15478 
15479   effect(DEF cr, USE op1, USE op2);
15480 
15481   ins_cost(INSN_COST);
15482   format %{ "cmp  $op1, $op2\t // ptr" %}
15483 
15484   ins_encode(aarch64_enc_cmpp(op1, op2));
15485 
15486   ins_pipe(icmp_reg_reg);
15487 %}
15488 
15489 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15490 %{
15491   match(Set cr (CmpN op1 op2));
15492 
15493   effect(DEF cr, USE op1, USE op2);
15494 
15495   ins_cost(INSN_COST);
15496   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15497 
15498   ins_encode(aarch64_enc_cmpn(op1, op2));
15499 
15500   ins_pipe(icmp_reg_reg);
15501 %}
15502 
15503 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15504 %{
15505   match(Set cr (CmpP op1 zero));
15506 
15507   effect(DEF cr, USE op1, USE zero);
15508 
15509   ins_cost(INSN_COST);
15510   format %{ "cmp  $op1, 0\t // ptr" %}
15511 
15512   ins_encode(aarch64_enc_testp(op1));
15513 
15514   ins_pipe(icmp_reg_imm);
15515 %}
15516 
15517 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15518 %{
15519   match(Set cr (CmpN op1 zero));
15520 
15521   effect(DEF cr, USE op1, USE zero);
15522 
15523   ins_cost(INSN_COST);
15524   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15525 
15526   ins_encode(aarch64_enc_testn(op1));
15527 
15528   ins_pipe(icmp_reg_imm);
15529 %}
15530 
15531 // FP comparisons
15532 //
15533 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15534 // using normal cmpOp. See declaration of rFlagsReg for details.
15535 
15536 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15537 %{
15538   match(Set cr (CmpF src1 src2));
15539 
15540   ins_cost(3 * INSN_COST);
15541   format %{ "fcmps $src1, $src2" %}
15542 
15543   ins_encode %{
15544     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15545   %}
15546 
15547   ins_pipe(pipe_class_compare);
15548 %}
15549 
15550 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15551 %{
15552   match(Set cr (CmpF src1 src2));
15553 
15554   ins_cost(3 * INSN_COST);
15555   format %{ "fcmps $src1, 0.0" %}
15556 
15557   ins_encode %{
15558     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15559   %}
15560 
15561   ins_pipe(pipe_class_compare);
15562 %}
15563 // FROM HERE
15564 
15565 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15566 %{
15567   match(Set cr (CmpD src1 src2));
15568 
15569   ins_cost(3 * INSN_COST);
15570   format %{ "fcmpd $src1, $src2" %}
15571 
15572   ins_encode %{
15573     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15574   %}
15575 
15576   ins_pipe(pipe_class_compare);
15577 %}
15578 
15579 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15580 %{
15581   match(Set cr (CmpD src1 src2));
15582 
15583   ins_cost(3 * INSN_COST);
15584   format %{ "fcmpd $src1, 0.0" %}
15585 
15586   ins_encode %{
15587     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15588   %}
15589 
15590   ins_pipe(pipe_class_compare);
15591 %}
15592 
15593 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15594 %{
15595   match(Set dst (CmpF3 src1 src2));
15596   effect(KILL cr);
15597 
15598   ins_cost(5 * INSN_COST);
15599   format %{ "fcmps $src1, $src2\n\t"
15600             "csinvw($dst, zr, zr, eq\n\t"
15601             "csnegw($dst, $dst, $dst, lt)"
15602   %}
15603 
15604   ins_encode %{
15605     Label done;
15606     FloatRegister s1 = as_FloatRegister($src1$$reg);
15607     FloatRegister s2 = as_FloatRegister($src2$$reg);
15608     Register d = as_Register($dst$$reg);
15609     __ fcmps(s1, s2);
15610     // installs 0 if EQ else -1
15611     __ csinvw(d, zr, zr, Assembler::EQ);
15612     // keeps -1 if less or unordered else installs 1
15613     __ csnegw(d, d, d, Assembler::LT);
15614     __ bind(done);
15615   %}
15616 
15617   ins_pipe(pipe_class_default);
15618 
15619 %}
15620 
15621 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15622 %{
15623   match(Set dst (CmpD3 src1 src2));
15624   effect(KILL cr);
15625 
15626   ins_cost(5 * INSN_COST);
15627   format %{ "fcmpd $src1, $src2\n\t"
15628             "csinvw($dst, zr, zr, eq\n\t"
15629             "csnegw($dst, $dst, $dst, lt)"
15630   %}
15631 
15632   ins_encode %{
15633     Label done;
15634     FloatRegister s1 = as_FloatRegister($src1$$reg);
15635     FloatRegister s2 = as_FloatRegister($src2$$reg);
15636     Register d = as_Register($dst$$reg);
15637     __ fcmpd(s1, s2);
15638     // installs 0 if EQ else -1
15639     __ csinvw(d, zr, zr, Assembler::EQ);
15640     // keeps -1 if less or unordered else installs 1
15641     __ csnegw(d, d, d, Assembler::LT);
15642     __ bind(done);
15643   %}
15644   ins_pipe(pipe_class_default);
15645 
15646 %}
15647 
15648 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15649 %{
15650   match(Set dst (CmpF3 src1 zero));
15651   effect(KILL cr);
15652 
15653   ins_cost(5 * INSN_COST);
15654   format %{ "fcmps $src1, 0.0\n\t"
15655             "csinvw($dst, zr, zr, eq\n\t"
15656             "csnegw($dst, $dst, $dst, lt)"
15657   %}
15658 
15659   ins_encode %{
15660     Label done;
15661     FloatRegister s1 = as_FloatRegister($src1$$reg);
15662     Register d = as_Register($dst$$reg);
15663     __ fcmps(s1, 0.0);
15664     // installs 0 if EQ else -1
15665     __ csinvw(d, zr, zr, Assembler::EQ);
15666     // keeps -1 if less or unordered else installs 1
15667     __ csnegw(d, d, d, Assembler::LT);
15668     __ bind(done);
15669   %}
15670 
15671   ins_pipe(pipe_class_default);
15672 
15673 %}
15674 
15675 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15676 %{
15677   match(Set dst (CmpD3 src1 zero));
15678   effect(KILL cr);
15679 
15680   ins_cost(5 * INSN_COST);
15681   format %{ "fcmpd $src1, 0.0\n\t"
15682             "csinvw($dst, zr, zr, eq\n\t"
15683             "csnegw($dst, $dst, $dst, lt)"
15684   %}
15685 
15686   ins_encode %{
15687     Label done;
15688     FloatRegister s1 = as_FloatRegister($src1$$reg);
15689     Register d = as_Register($dst$$reg);
15690     __ fcmpd(s1, 0.0);
15691     // installs 0 if EQ else -1
15692     __ csinvw(d, zr, zr, Assembler::EQ);
15693     // keeps -1 if less or unordered else installs 1
15694     __ csnegw(d, d, d, Assembler::LT);
15695     __ bind(done);
15696   %}
15697   ins_pipe(pipe_class_default);
15698 
15699 %}
15700 
15701 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15702 %{
15703   match(Set dst (CmpLTMask p q));
15704   effect(KILL cr);
15705 
15706   ins_cost(3 * INSN_COST);
15707 
15708   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15709             "csetw $dst, lt\n\t"
15710             "subw $dst, zr, $dst"
15711   %}
15712 
15713   ins_encode %{
15714     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15715     __ csetw(as_Register($dst$$reg), Assembler::LT);
15716     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15717   %}
15718 
15719   ins_pipe(ialu_reg_reg);
15720 %}
15721 
15722 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15723 %{
15724   match(Set dst (CmpLTMask src zero));
15725   effect(KILL cr);
15726 
15727   ins_cost(INSN_COST);
15728 
15729   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15730 
15731   ins_encode %{
15732     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15733   %}
15734 
15735   ins_pipe(ialu_reg_shift);
15736 %}
15737 
15738 // ============================================================================
15739 // Max and Min
15740 
15741 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15742 %{
15743   effect( DEF dst, USE src1, USE src2, USE cr );
15744 
15745   ins_cost(INSN_COST * 2);
15746   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15747 
15748   ins_encode %{
15749     __ cselw(as_Register($dst$$reg),
15750              as_Register($src1$$reg),
15751              as_Register($src2$$reg),
15752              Assembler::LT);
15753   %}
15754 
15755   ins_pipe(icond_reg_reg);
15756 %}
15757 
15758 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15759 %{
15760   match(Set dst (MinI src1 src2));
15761   ins_cost(INSN_COST * 3);
15762 
15763   expand %{
15764     rFlagsReg cr;
15765     compI_reg_reg(cr, src1, src2);
15766     cmovI_reg_reg_lt(dst, src1, src2, cr);
15767   %}
15768 
15769 %}
15770 // FROM HERE
15771 
15772 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15773 %{
15774   effect( DEF dst, USE src1, USE src2, USE cr );
15775 
15776   ins_cost(INSN_COST * 2);
15777   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15778 
15779   ins_encode %{
15780     __ cselw(as_Register($dst$$reg),
15781              as_Register($src1$$reg),
15782              as_Register($src2$$reg),
15783              Assembler::GT);
15784   %}
15785 
15786   ins_pipe(icond_reg_reg);
15787 %}
15788 
15789 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15790 %{
15791   match(Set dst (MaxI src1 src2));
15792   ins_cost(INSN_COST * 3);
15793   expand %{
15794     rFlagsReg cr;
15795     compI_reg_reg(cr, src1, src2);
15796     cmovI_reg_reg_gt(dst, src1, src2, cr);
15797   %}
15798 %}
15799 
15800 // ============================================================================
15801 // Branch Instructions
15802 
15803 // Direct Branch.
15804 instruct branch(label lbl)
15805 %{
15806   match(Goto);
15807 
15808   effect(USE lbl);
15809 
15810   ins_cost(BRANCH_COST);
15811   format %{ "b  $lbl" %}
15812 
15813   ins_encode(aarch64_enc_b(lbl));
15814 
15815   ins_pipe(pipe_branch);
15816 %}
15817 
15818 // Conditional Near Branch
15819 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15820 %{
15821   // Same match rule as `branchConFar'.
15822   match(If cmp cr);
15823 
15824   effect(USE lbl);
15825 
15826   ins_cost(BRANCH_COST);
15827   // If set to 1 this indicates that the current instruction is a
15828   // short variant of a long branch. This avoids using this
15829   // instruction in first-pass matching. It will then only be used in
15830   // the `Shorten_branches' pass.
15831   // ins_short_branch(1);
15832   format %{ "b$cmp  $lbl" %}
15833 
15834   ins_encode(aarch64_enc_br_con(cmp, lbl));
15835 
15836   ins_pipe(pipe_branch_cond);
15837 %}
15838 
15839 // Conditional Near Branch Unsigned
15840 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15841 %{
15842   // Same match rule as `branchConFar'.
15843   match(If cmp cr);
15844 
15845   effect(USE lbl);
15846 
15847   ins_cost(BRANCH_COST);
15848   // If set to 1 this indicates that the current instruction is a
15849   // short variant of a long branch. This avoids using this
15850   // instruction in first-pass matching. It will then only be used in
15851   // the `Shorten_branches' pass.
15852   // ins_short_branch(1);
15853   format %{ "b$cmp  $lbl\t# unsigned" %}
15854 
15855   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15856 
15857   ins_pipe(pipe_branch_cond);
15858 %}
15859 
15860 // Make use of CBZ and CBNZ.  These instructions, as well as being
15861 // shorter than (cmp; branch), have the additional benefit of not
15862 // killing the flags.
15863 
15864 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15865   match(If cmp (CmpI op1 op2));
15866   effect(USE labl);
15867 
15868   ins_cost(BRANCH_COST);
15869   format %{ "cbw$cmp   $op1, $labl" %}
15870   ins_encode %{
15871     Label* L = $labl$$label;
15872     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15873     if (cond == Assembler::EQ)
15874       __ cbzw($op1$$Register, *L);
15875     else
15876       __ cbnzw($op1$$Register, *L);
15877   %}
15878   ins_pipe(pipe_cmp_branch);
15879 %}
15880 
15881 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15882   match(If cmp (CmpL op1 op2));
15883   effect(USE labl);
15884 
15885   ins_cost(BRANCH_COST);
15886   format %{ "cb$cmp   $op1, $labl" %}
15887   ins_encode %{
15888     Label* L = $labl$$label;
15889     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15890     if (cond == Assembler::EQ)
15891       __ cbz($op1$$Register, *L);
15892     else
15893       __ cbnz($op1$$Register, *L);
15894   %}
15895   ins_pipe(pipe_cmp_branch);
15896 %}
15897 
15898 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15899   match(If cmp (CmpP op1 op2));
15900   effect(USE labl);
15901 
15902   ins_cost(BRANCH_COST);
15903   format %{ "cb$cmp   $op1, $labl" %}
15904   ins_encode %{
15905     Label* L = $labl$$label;
15906     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15907     if (cond == Assembler::EQ)
15908       __ cbz($op1$$Register, *L);
15909     else
15910       __ cbnz($op1$$Register, *L);
15911   %}
15912   ins_pipe(pipe_cmp_branch);
15913 %}
15914 
15915 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15916   match(If cmp (CmpN op1 op2));
15917   effect(USE labl);
15918 
15919   ins_cost(BRANCH_COST);
15920   format %{ "cbw$cmp   $op1, $labl" %}
15921   ins_encode %{
15922     Label* L = $labl$$label;
15923     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15924     if (cond == Assembler::EQ)
15925       __ cbzw($op1$$Register, *L);
15926     else
15927       __ cbnzw($op1$$Register, *L);
15928   %}
15929   ins_pipe(pipe_cmp_branch);
15930 %}
15931 
15932 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15933   match(If cmp (CmpP (DecodeN oop) zero));
15934   effect(USE labl);
15935 
15936   ins_cost(BRANCH_COST);
15937   format %{ "cb$cmp   $oop, $labl" %}
15938   ins_encode %{
15939     Label* L = $labl$$label;
15940     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15941     if (cond == Assembler::EQ)
15942       __ cbzw($oop$$Register, *L);
15943     else
15944       __ cbnzw($oop$$Register, *L);
15945   %}
15946   ins_pipe(pipe_cmp_branch);
15947 %}
15948 
15949 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15950   match(If cmp (CmpU op1 op2));
15951   effect(USE labl);
15952 
15953   ins_cost(BRANCH_COST);
15954   format %{ "cbw$cmp   $op1, $labl" %}
15955   ins_encode %{
15956     Label* L = $labl$$label;
15957     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15958     if (cond == Assembler::EQ || cond == Assembler::LS)
15959       __ cbzw($op1$$Register, *L);
15960     else
15961       __ cbnzw($op1$$Register, *L);
15962   %}
15963   ins_pipe(pipe_cmp_branch);
15964 %}
15965 
15966 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15967   match(If cmp (CmpUL op1 op2));
15968   effect(USE labl);
15969 
15970   ins_cost(BRANCH_COST);
15971   format %{ "cb$cmp   $op1, $labl" %}
15972   ins_encode %{
15973     Label* L = $labl$$label;
15974     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15975     if (cond == Assembler::EQ || cond == Assembler::LS)
15976       __ cbz($op1$$Register, *L);
15977     else
15978       __ cbnz($op1$$Register, *L);
15979   %}
15980   ins_pipe(pipe_cmp_branch);
15981 %}
15982 
15983 // Test bit and Branch
15984 
15985 // Patterns for short (< 32KiB) variants
15986 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15987   match(If cmp (CmpL op1 op2));
15988   effect(USE labl);
15989 
15990   ins_cost(BRANCH_COST);
15991   format %{ "cb$cmp   $op1, $labl # long" %}
15992   ins_encode %{
15993     Label* L = $labl$$label;
15994     Assembler::Condition cond =
15995       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15996     __ tbr(cond, $op1$$Register, 63, *L);
15997   %}
15998   ins_pipe(pipe_cmp_branch);
15999   ins_short_branch(1);
16000 %}
16001 
16002 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16003   match(If cmp (CmpI op1 op2));
16004   effect(USE labl);
16005 
16006   ins_cost(BRANCH_COST);
16007   format %{ "cb$cmp   $op1, $labl # int" %}
16008   ins_encode %{
16009     Label* L = $labl$$label;
16010     Assembler::Condition cond =
16011       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16012     __ tbr(cond, $op1$$Register, 31, *L);
16013   %}
16014   ins_pipe(pipe_cmp_branch);
16015   ins_short_branch(1);
16016 %}
16017 
16018 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16019   match(If cmp (CmpL (AndL op1 op2) op3));
16020   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16021   effect(USE labl);
16022 
16023   ins_cost(BRANCH_COST);
16024   format %{ "tb$cmp   $op1, $op2, $labl" %}
16025   ins_encode %{
16026     Label* L = $labl$$label;
16027     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16028     int bit = exact_log2_long($op2$$constant);
16029     __ tbr(cond, $op1$$Register, bit, *L);
16030   %}
16031   ins_pipe(pipe_cmp_branch);
16032   ins_short_branch(1);
16033 %}
16034 
16035 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16036   match(If cmp (CmpI (AndI op1 op2) op3));
16037   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16038   effect(USE labl);
16039 
16040   ins_cost(BRANCH_COST);
16041   format %{ "tb$cmp   $op1, $op2, $labl" %}
16042   ins_encode %{
16043     Label* L = $labl$$label;
16044     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16045     int bit = exact_log2((juint)$op2$$constant);
16046     __ tbr(cond, $op1$$Register, bit, *L);
16047   %}
16048   ins_pipe(pipe_cmp_branch);
16049   ins_short_branch(1);
16050 %}
16051 
16052 // And far variants
16053 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16054   match(If cmp (CmpL op1 op2));
16055   effect(USE labl);
16056 
16057   ins_cost(BRANCH_COST);
16058   format %{ "cb$cmp   $op1, $labl # long" %}
16059   ins_encode %{
16060     Label* L = $labl$$label;
16061     Assembler::Condition cond =
16062       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16063     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16064   %}
16065   ins_pipe(pipe_cmp_branch);
16066 %}
16067 
16068 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16069   match(If cmp (CmpI op1 op2));
16070   effect(USE labl);
16071 
16072   ins_cost(BRANCH_COST);
16073   format %{ "cb$cmp   $op1, $labl # int" %}
16074   ins_encode %{
16075     Label* L = $labl$$label;
16076     Assembler::Condition cond =
16077       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16078     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16079   %}
16080   ins_pipe(pipe_cmp_branch);
16081 %}
16082 
16083 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16084   match(If cmp (CmpL (AndL op1 op2) op3));
16085   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16086   effect(USE labl);
16087 
16088   ins_cost(BRANCH_COST);
16089   format %{ "tb$cmp   $op1, $op2, $labl" %}
16090   ins_encode %{
16091     Label* L = $labl$$label;
16092     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16093     int bit = exact_log2_long($op2$$constant);
16094     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16095   %}
16096   ins_pipe(pipe_cmp_branch);
16097 %}
16098 
16099 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16100   match(If cmp (CmpI (AndI op1 op2) op3));
16101   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16102   effect(USE labl);
16103 
16104   ins_cost(BRANCH_COST);
16105   format %{ "tb$cmp   $op1, $op2, $labl" %}
16106   ins_encode %{
16107     Label* L = $labl$$label;
16108     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16109     int bit = exact_log2((juint)$op2$$constant);
16110     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16111   %}
16112   ins_pipe(pipe_cmp_branch);
16113 %}
16114 
16115 // Test bits
16116 
16117 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16118   match(Set cr (CmpL (AndL op1 op2) op3));
16119   predicate(Assembler::operand_valid_for_logical_immediate
16120             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16121 
16122   ins_cost(INSN_COST);
16123   format %{ "tst $op1, $op2 # long" %}
16124   ins_encode %{
16125     __ tst($op1$$Register, $op2$$constant);
16126   %}
16127   ins_pipe(ialu_reg_reg);
16128 %}
16129 
16130 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16131   match(Set cr (CmpI (AndI op1 op2) op3));
16132   predicate(Assembler::operand_valid_for_logical_immediate
16133             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16134 
16135   ins_cost(INSN_COST);
16136   format %{ "tst $op1, $op2 # int" %}
16137   ins_encode %{
16138     __ tstw($op1$$Register, $op2$$constant);
16139   %}
16140   ins_pipe(ialu_reg_reg);
16141 %}
16142 
16143 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16144   match(Set cr (CmpL (AndL op1 op2) op3));
16145 
16146   ins_cost(INSN_COST);
16147   format %{ "tst $op1, $op2 # long" %}
16148   ins_encode %{
16149     __ tst($op1$$Register, $op2$$Register);
16150   %}
16151   ins_pipe(ialu_reg_reg);
16152 %}
16153 
16154 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16155   match(Set cr (CmpI (AndI op1 op2) op3));
16156 
16157   ins_cost(INSN_COST);
16158   format %{ "tstw $op1, $op2 # int" %}
16159   ins_encode %{
16160     __ tstw($op1$$Register, $op2$$Register);
16161   %}
16162   ins_pipe(ialu_reg_reg);
16163 %}
16164 
16165 
16166 // Conditional Far Branch
16167 // Conditional Far Branch Unsigned
16168 // TODO: fixme
16169 
16170 // counted loop end branch near
16171 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16172 %{
16173   match(CountedLoopEnd cmp cr);
16174 
16175   effect(USE lbl);
16176 
16177   ins_cost(BRANCH_COST);
16178   // short variant.
16179   // ins_short_branch(1);
16180   format %{ "b$cmp $lbl \t// counted loop end" %}
16181 
16182   ins_encode(aarch64_enc_br_con(cmp, lbl));
16183 
16184   ins_pipe(pipe_branch);
16185 %}
16186 
16187 // counted loop end branch near Unsigned
16188 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16189 %{
16190   match(CountedLoopEnd cmp cr);
16191 
16192   effect(USE lbl);
16193 
16194   ins_cost(BRANCH_COST);
16195   // short variant.
16196   // ins_short_branch(1);
16197   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16198 
16199   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16200 
16201   ins_pipe(pipe_branch);
16202 %}
16203 
16204 // counted loop end branch far
16205 // counted loop end branch far unsigned
16206 // TODO: fixme
16207 
16208 // ============================================================================
16209 // inlined locking and unlocking
16210 
16211 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16212 %{
16213   match(Set cr (FastLock object box));
16214   effect(TEMP tmp, TEMP tmp2);
16215 
16216   // TODO
16217   // identify correct cost
16218   ins_cost(5 * INSN_COST);
16219   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16220 
16221   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16222 
16223   ins_pipe(pipe_serial);
16224 %}
16225 
16226 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16227 %{
16228   match(Set cr (FastUnlock object box));
16229   effect(TEMP tmp, TEMP tmp2);
16230 
16231   ins_cost(5 * INSN_COST);
16232   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16233 
16234   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16235 
16236   ins_pipe(pipe_serial);
16237 %}
16238 
16239 
16240 // ============================================================================
16241 // Safepoint Instructions
16242 
16243 // TODO
16244 // provide a near and far version of this code
16245 
16246 instruct safePoint(rFlagsReg cr, iRegP poll)
16247 %{
16248   match(SafePoint poll);
16249   effect(KILL cr);
16250 
16251   format %{
16252     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16253   %}
16254   ins_encode %{
16255     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16256   %}
16257   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16258 %}
16259 
16260 
16261 // ============================================================================
16262 // Procedure Call/Return Instructions
16263 
16264 // Call Java Static Instruction
16265 
16266 instruct CallStaticJavaDirect(method meth)
16267 %{
16268   match(CallStaticJava);
16269 
16270   effect(USE meth);
16271 
16272   ins_cost(CALL_COST);
16273 
16274   format %{ "call,static $meth \t// ==> " %}
16275 
16276   ins_encode(aarch64_enc_java_static_call(meth),
16277              aarch64_enc_call_epilog);
16278 
16279   ins_pipe(pipe_class_call);
16280 %}
16281 
16282 // TO HERE
16283 
16284 // Call Java Dynamic Instruction
16285 instruct CallDynamicJavaDirect(method meth)
16286 %{
16287   match(CallDynamicJava);
16288 
16289   effect(USE meth);
16290 
16291   ins_cost(CALL_COST);
16292 
16293   format %{ "CALL,dynamic $meth \t// ==> " %}
16294 
16295   ins_encode(aarch64_enc_java_dynamic_call(meth),
16296              aarch64_enc_call_epilog);
16297 
16298   ins_pipe(pipe_class_call);
16299 %}
16300 
16301 // Call Runtime Instruction
16302 
16303 instruct CallRuntimeDirect(method meth)
16304 %{
16305   match(CallRuntime);
16306 
16307   effect(USE meth);
16308 
16309   ins_cost(CALL_COST);
16310 
16311   format %{ "CALL, runtime $meth" %}
16312 
16313   ins_encode( aarch64_enc_java_to_runtime(meth) );
16314 
16315   ins_pipe(pipe_class_call);
16316 %}
16317 
16318 // Call Runtime Instruction
16319 
16320 instruct CallLeafDirect(method meth)
16321 %{
16322   match(CallLeaf);
16323 
16324   effect(USE meth);
16325 
16326   ins_cost(CALL_COST);
16327 
16328   format %{ "CALL, runtime leaf $meth" %}
16329 
16330   ins_encode( aarch64_enc_java_to_runtime(meth) );
16331 
16332   ins_pipe(pipe_class_call);
16333 %}
16334 
16335 // Call Runtime Instruction
16336 
16337 instruct CallLeafNoFPDirect(method meth)
16338 %{
16339   match(CallLeafNoFP);
16340 
16341   effect(USE meth);
16342 
16343   ins_cost(CALL_COST);
16344 
16345   format %{ "CALL, runtime leaf nofp $meth" %}
16346 
16347   ins_encode( aarch64_enc_java_to_runtime(meth) );
16348 
16349   ins_pipe(pipe_class_call);
16350 %}
16351 
16352 instruct CallNativeDirect(method meth)
16353 %{
16354   match(CallNative);
16355 
16356   effect(USE meth);
16357 
16358   ins_cost(CALL_COST);
16359 
16360   format %{ "CALL, native $meth" %}
16361 
16362   ins_encode( aarch64_enc_java_to_runtime(meth) );
16363 
16364   ins_pipe(pipe_class_call);
16365 %}
16366 
16367 // Tail Call; Jump from runtime stub to Java code.
16368 // Also known as an 'interprocedural jump'.
16369 // Target of jump will eventually return to caller.
16370 // TailJump below removes the return address.
16371 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16372 %{
16373   match(TailCall jump_target method_ptr);
16374 
16375   ins_cost(CALL_COST);
16376 
16377   format %{ "br $jump_target\t# $method_ptr holds method" %}
16378 
16379   ins_encode(aarch64_enc_tail_call(jump_target));
16380 
16381   ins_pipe(pipe_class_call);
16382 %}
16383 
16384 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16385 %{
16386   match(TailJump jump_target ex_oop);
16387 
16388   ins_cost(CALL_COST);
16389 
16390   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16391 
16392   ins_encode(aarch64_enc_tail_jmp(jump_target));
16393 
16394   ins_pipe(pipe_class_call);
16395 %}
16396 
16397 // Create exception oop: created by stack-crawling runtime code.
16398 // Created exception is now available to this handler, and is setup
16399 // just prior to jumping to this handler. No code emitted.
16400 // TODO check
16401 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16402 instruct CreateException(iRegP_R0 ex_oop)
16403 %{
16404   match(Set ex_oop (CreateEx));
16405 
16406   format %{ " -- \t// exception oop; no code emitted" %}
16407 
16408   size(0);
16409 
16410   ins_encode( /*empty*/ );
16411 
16412   ins_pipe(pipe_class_empty);
16413 %}
16414 
16415 // Rethrow exception: The exception oop will come in the first
16416 // argument position. Then JUMP (not call) to the rethrow stub code.
16417 instruct RethrowException() %{
16418   match(Rethrow);
16419   ins_cost(CALL_COST);
16420 
16421   format %{ "b rethrow_stub" %}
16422 
16423   ins_encode( aarch64_enc_rethrow() );
16424 
16425   ins_pipe(pipe_class_call);
16426 %}
16427 
16428 
16429 // Return Instruction
16430 // epilog node loads ret address into lr as part of frame pop
16431 instruct Ret()
16432 %{
16433   match(Return);
16434 
16435   format %{ "ret\t// return register" %}
16436 
16437   ins_encode( aarch64_enc_ret() );
16438 
16439   ins_pipe(pipe_branch);
16440 %}
16441 
16442 // Die now.
16443 instruct ShouldNotReachHere() %{
16444   match(Halt);
16445 
16446   ins_cost(CALL_COST);
16447   format %{ "ShouldNotReachHere" %}
16448 
16449   ins_encode %{
16450     if (is_reachable()) {
16451       __ stop(_halt_reason);
16452     }
16453   %}
16454 
16455   ins_pipe(pipe_class_default);
16456 %}
16457 
16458 // ============================================================================
16459 // Partial Subtype Check
16460 //
16461 // superklass array for an instance of the superklass.  Set a hidden
16462 // internal cache on a hit (cache is checked with exposed code in
16463 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16464 // encoding ALSO sets flags.
16465 
16466 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16467 %{
16468   match(Set result (PartialSubtypeCheck sub super));
16469   effect(KILL cr, KILL temp);
16470 
16471   ins_cost(1100);  // slightly larger than the next version
16472   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16473 
16474   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16475 
16476   opcode(0x1); // Force zero of result reg on hit
16477 
16478   ins_pipe(pipe_class_memory);
16479 %}
16480 
16481 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16482 %{
16483   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16484   effect(KILL temp, KILL result);
16485 
16486   ins_cost(1100);  // slightly larger than the next version
16487   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16488 
16489   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16490 
16491   opcode(0x0); // Don't zero result reg on hit
16492 
16493   ins_pipe(pipe_class_memory);
16494 %}
16495 
16496 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16497                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16498 %{
16499   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16500   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16501   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16502 
16503   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16504   ins_encode %{
16505     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16506     __ string_compare($str1$$Register, $str2$$Register,
16507                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16508                       $tmp1$$Register, $tmp2$$Register,
16509                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16510   %}
16511   ins_pipe(pipe_class_memory);
16512 %}
16513 
16514 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16515                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16516 %{
16517   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16518   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16519   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16520 
16521   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16522   ins_encode %{
16523     __ string_compare($str1$$Register, $str2$$Register,
16524                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16525                       $tmp1$$Register, $tmp2$$Register,
16526                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16527   %}
16528   ins_pipe(pipe_class_memory);
16529 %}
16530 
16531 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16532                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16533                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16534 %{
16535   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16536   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16537   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16538          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16539 
16540   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16541   ins_encode %{
16542     __ string_compare($str1$$Register, $str2$$Register,
16543                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16544                       $tmp1$$Register, $tmp2$$Register,
16545                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16546                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16547   %}
16548   ins_pipe(pipe_class_memory);
16549 %}
16550 
16551 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16552                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16553                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16554 %{
16555   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16556   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16557   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16558          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16559 
16560   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16561   ins_encode %{
16562     __ string_compare($str1$$Register, $str2$$Register,
16563                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16564                       $tmp1$$Register, $tmp2$$Register,
16565                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16566                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16567   %}
16568   ins_pipe(pipe_class_memory);
16569 %}
16570 
16571 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16572        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16573        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16574 %{
16575   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16576   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16577   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16578          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16579   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16580 
16581   ins_encode %{
16582     __ string_indexof($str1$$Register, $str2$$Register,
16583                       $cnt1$$Register, $cnt2$$Register,
16584                       $tmp1$$Register, $tmp2$$Register,
16585                       $tmp3$$Register, $tmp4$$Register,
16586                       $tmp5$$Register, $tmp6$$Register,
16587                       -1, $result$$Register, StrIntrinsicNode::UU);
16588   %}
16589   ins_pipe(pipe_class_memory);
16590 %}
16591 
16592 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16593        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16594        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16595 %{
16596   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16597   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16598   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16599          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16600   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16601 
16602   ins_encode %{
16603     __ string_indexof($str1$$Register, $str2$$Register,
16604                       $cnt1$$Register, $cnt2$$Register,
16605                       $tmp1$$Register, $tmp2$$Register,
16606                       $tmp3$$Register, $tmp4$$Register,
16607                       $tmp5$$Register, $tmp6$$Register,
16608                       -1, $result$$Register, StrIntrinsicNode::LL);
16609   %}
16610   ins_pipe(pipe_class_memory);
16611 %}
16612 
16613 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16614        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16615        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16616 %{
16617   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16618   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16619   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16620          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16621   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16622 
16623   ins_encode %{
16624     __ string_indexof($str1$$Register, $str2$$Register,
16625                       $cnt1$$Register, $cnt2$$Register,
16626                       $tmp1$$Register, $tmp2$$Register,
16627                       $tmp3$$Register, $tmp4$$Register,
16628                       $tmp5$$Register, $tmp6$$Register,
16629                       -1, $result$$Register, StrIntrinsicNode::UL);
16630   %}
16631   ins_pipe(pipe_class_memory);
16632 %}
16633 
16634 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16635                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16636                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16637 %{
16638   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16639   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16640   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16641          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16642   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16643 
16644   ins_encode %{
16645     int icnt2 = (int)$int_cnt2$$constant;
16646     __ string_indexof($str1$$Register, $str2$$Register,
16647                       $cnt1$$Register, zr,
16648                       $tmp1$$Register, $tmp2$$Register,
16649                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16650                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16651   %}
16652   ins_pipe(pipe_class_memory);
16653 %}
16654 
16655 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16656                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16657                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16658 %{
16659   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16660   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16661   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16662          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16663   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16664 
16665   ins_encode %{
16666     int icnt2 = (int)$int_cnt2$$constant;
16667     __ string_indexof($str1$$Register, $str2$$Register,
16668                       $cnt1$$Register, zr,
16669                       $tmp1$$Register, $tmp2$$Register,
16670                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16671                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16672   %}
16673   ins_pipe(pipe_class_memory);
16674 %}
16675 
16676 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16677                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16678                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16679 %{
16680   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16681   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16682   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16683          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16684   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16685 
16686   ins_encode %{
16687     int icnt2 = (int)$int_cnt2$$constant;
16688     __ string_indexof($str1$$Register, $str2$$Register,
16689                       $cnt1$$Register, zr,
16690                       $tmp1$$Register, $tmp2$$Register,
16691                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16692                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16693   %}
16694   ins_pipe(pipe_class_memory);
16695 %}
16696 
16697 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16698                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16699                              iRegINoSp tmp3, rFlagsReg cr)
16700 %{
16701   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16702   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16703   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16704          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16705 
16706   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16707 
16708   ins_encode %{
16709     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16710                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16711                            $tmp3$$Register);
16712   %}
16713   ins_pipe(pipe_class_memory);
16714 %}
16715 
16716 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16717                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16718                               iRegINoSp tmp3, rFlagsReg cr)
16719 %{
16720   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16721   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16722   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16723          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16724 
16725   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16726 
16727   ins_encode %{
16728     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16729                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16730                             $tmp3$$Register);
16731   %}
16732   ins_pipe(pipe_class_memory);
16733 %}
16734 
16735 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16736                         iRegI_R0 result, rFlagsReg cr)
16737 %{
16738   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16739   match(Set result (StrEquals (Binary str1 str2) cnt));
16740   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16741 
16742   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16743   ins_encode %{
16744     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16745     __ string_equals($str1$$Register, $str2$$Register,
16746                      $result$$Register, $cnt$$Register, 1);
16747   %}
16748   ins_pipe(pipe_class_memory);
16749 %}
16750 
16751 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16752                         iRegI_R0 result, rFlagsReg cr)
16753 %{
16754   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16755   match(Set result (StrEquals (Binary str1 str2) cnt));
16756   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16757 
16758   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16759   ins_encode %{
16760     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16761     __ string_equals($str1$$Register, $str2$$Register,
16762                      $result$$Register, $cnt$$Register, 2);
16763   %}
16764   ins_pipe(pipe_class_memory);
16765 %}
16766 
16767 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16768                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16769                        iRegP_R10 tmp, rFlagsReg cr)
16770 %{
16771   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16772   match(Set result (AryEq ary1 ary2));
16773   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16774 
16775   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16776   ins_encode %{
16777     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16778                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16779                                    $result$$Register, $tmp$$Register, 1);
16780     if (tpc == NULL) {
16781       ciEnv::current()->record_failure("CodeCache is full");
16782       return;
16783     }
16784   %}
16785   ins_pipe(pipe_class_memory);
16786 %}
16787 
16788 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16789                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16790                        iRegP_R10 tmp, rFlagsReg cr)
16791 %{
16792   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16793   match(Set result (AryEq ary1 ary2));
16794   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16795 
16796   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16797   ins_encode %{
16798     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16799                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16800                                    $result$$Register, $tmp$$Register, 2);
16801     if (tpc == NULL) {
16802       ciEnv::current()->record_failure("CodeCache is full");
16803       return;
16804     }
16805   %}
16806   ins_pipe(pipe_class_memory);
16807 %}
16808 
16809 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16810 %{
16811   match(Set result (HasNegatives ary1 len));
16812   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16813   format %{ "has negatives byte[] $ary1,$len -> $result" %}
16814   ins_encode %{
16815     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
16816     if (tpc == NULL) {
16817       ciEnv::current()->record_failure("CodeCache is full");
16818       return;
16819     }
16820   %}
16821   ins_pipe( pipe_slow );
16822 %}
16823 
16824 // fast char[] to byte[] compression
16825 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16826                          vRegD_V0 tmp1, vRegD_V1 tmp2,
16827                          vRegD_V2 tmp3, vRegD_V3 tmp4,
16828                          iRegI_R0 result, rFlagsReg cr)
16829 %{
16830   match(Set result (StrCompressedCopy src (Binary dst len)));
16831   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16832 
16833   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
16834   ins_encode %{
16835     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16836                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16837                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
16838                            $result$$Register);
16839   %}
16840   ins_pipe( pipe_slow );
16841 %}
16842 
16843 // fast byte[] to char[] inflation
16844 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16845                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16846 %{
16847   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16848   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16849 
16850   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16851   ins_encode %{
16852     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16853                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16854                                         $tmp3$$FloatRegister, $tmp4$$Register);
16855     if (tpc == NULL) {
16856       ciEnv::current()->record_failure("CodeCache is full");
16857       return;
16858     }
16859   %}
16860   ins_pipe(pipe_class_memory);
16861 %}
16862 
16863 // encode char[] to byte[] in ISO_8859_1
16864 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16865                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
16866                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
16867                           iRegI_R0 result, rFlagsReg cr)
16868 %{
16869   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16870   match(Set result (EncodeISOArray src (Binary dst len)));
16871   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
16872          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
16873 
16874   format %{ "Encode array $src,$dst,$len -> $result" %}
16875   ins_encode %{
16876     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16877          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
16878          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
16879   %}
16880   ins_pipe( pipe_class_memory );
16881 %}
16882 
16883 // ============================================================================
16884 // This name is KNOWN by the ADLC and cannot be changed.
16885 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16886 // for this guy.
16887 instruct tlsLoadP(thread_RegP dst)
16888 %{
16889   match(Set dst (ThreadLocal));
16890 
16891   ins_cost(0);
16892 
16893   format %{ " -- \t// $dst=Thread::current(), empty" %}
16894 
16895   size(0);
16896 
16897   ins_encode( /*empty*/ );
16898 
16899   ins_pipe(pipe_class_empty);
16900 %}
16901 
16902 //----------PEEPHOLE RULES-----------------------------------------------------
16903 // These must follow all instruction definitions as they use the names
16904 // defined in the instructions definitions.
16905 //
16906 // peepmatch ( root_instr_name [preceding_instruction]* );
16907 //
16908 // peepconstraint %{
16909 // (instruction_number.operand_name relational_op instruction_number.operand_name
16910 //  [, ...] );
16911 // // instruction numbers are zero-based using left to right order in peepmatch
16912 //
16913 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
16914 // // provide an instruction_number.operand_name for each operand that appears
16915 // // in the replacement instruction's match rule
16916 //
16917 // ---------VM FLAGS---------------------------------------------------------
16918 //
16919 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16920 //
16921 // Each peephole rule is given an identifying number starting with zero and
16922 // increasing by one in the order seen by the parser.  An individual peephole
16923 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16924 // on the command-line.
16925 //
16926 // ---------CURRENT LIMITATIONS----------------------------------------------
16927 //
16928 // Only match adjacent instructions in same basic block
16929 // Only equality constraints
16930 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16931 // Only one replacement instruction
16932 //
16933 // ---------EXAMPLE----------------------------------------------------------
16934 //
16935 // // pertinent parts of existing instructions in architecture description
16936 // instruct movI(iRegINoSp dst, iRegI src)
16937 // %{
16938 //   match(Set dst (CopyI src));
16939 // %}
16940 //
16941 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16942 // %{
16943 //   match(Set dst (AddI dst src));
16944 //   effect(KILL cr);
16945 // %}
16946 //
16947 // // Change (inc mov) to lea
16948 // peephole %{
16949 //   // increment preceeded by register-register move
16950 //   peepmatch ( incI_iReg movI );
16951 //   // require that the destination register of the increment
16952 //   // match the destination register of the move
16953 //   peepconstraint ( 0.dst == 1.dst );
16954 //   // construct a replacement instruction that sets
16955 //   // the destination to ( move's source register + one )
16956 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16957 // %}
16958 //
16959 
16960 // Implementation no longer uses movX instructions since
16961 // machine-independent system no longer uses CopyX nodes.
16962 //
16963 // peephole
16964 // %{
16965 //   peepmatch (incI_iReg movI);
16966 //   peepconstraint (0.dst == 1.dst);
16967 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16968 // %}
16969 
16970 // peephole
16971 // %{
16972 //   peepmatch (decI_iReg movI);
16973 //   peepconstraint (0.dst == 1.dst);
16974 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16975 // %}
16976 
16977 // peephole
16978 // %{
16979 //   peepmatch (addI_iReg_imm movI);
16980 //   peepconstraint (0.dst == 1.dst);
16981 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16982 // %}
16983 
16984 // peephole
16985 // %{
16986 //   peepmatch (incL_iReg movL);
16987 //   peepconstraint (0.dst == 1.dst);
16988 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16989 // %}
16990 
16991 // peephole
16992 // %{
16993 //   peepmatch (decL_iReg movL);
16994 //   peepconstraint (0.dst == 1.dst);
16995 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16996 // %}
16997 
16998 // peephole
16999 // %{
17000 //   peepmatch (addL_iReg_imm movL);
17001 //   peepconstraint (0.dst == 1.dst);
17002 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17003 // %}
17004 
17005 // peephole
17006 // %{
17007 //   peepmatch (addP_iReg_imm movP);
17008 //   peepconstraint (0.dst == 1.dst);
17009 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17010 // %}
17011 
17012 // // Change load of spilled value to only a spill
17013 // instruct storeI(memory mem, iRegI src)
17014 // %{
17015 //   match(Set mem (StoreI mem src));
17016 // %}
17017 //
17018 // instruct loadI(iRegINoSp dst, memory mem)
17019 // %{
17020 //   match(Set dst (LoadI mem));
17021 // %}
17022 //
17023 
17024 //----------SMARTSPILL RULES---------------------------------------------------
17025 // These must follow all instruction definitions as they use the names
17026 // defined in the instructions definitions.
17027 
17028 // Local Variables:
17029 // mode: c++
17030 // End: